[IGAWorks] Natural Language Processing

import pandas as pd
aud = pd.read_csv("data/[label] aud_wo_install&house.csv")
aud.head(3)

def short(x):
    return x[1::2]

def long(x):
    return x[::2]

aud["cc_list"] = aud["cate_code"].str.findall(r"[\w']+")
aud["short"] = aud["cc_list"].apply(short)
aud["long"] = aud["cc_list"].apply(long)
aud['counts'] = aud['long'].str.len() 

import seaborn as sns
import matplotlib.pyplot as plt

f, (ax_box, ax_hist) = plt.subplots(2, sharex=True, gridspec_kw={"height_ratios": (.15, .85)})
 
sns.boxplot(aud.counts, ax=ax_box)
sns.distplot(aud.counts, ax=ax_hist)
 
ax_box.set(xlabel='')

from collections import Counter
from  itertools import chain

vc = pd.Series(Counter(chain(*aud.long))).sort_index().rename_axis('value').reset_index(name='counts')
vc = vc.sort_values(by =['counts'] , ascending = False)
vc['ratio'] = vc['counts'] / len(aud)
vc.head(3)

f, (ax_box, ax_hist) = plt.subplots(2, sharex=True, gridspec_kw={"height_ratios": (.15, .85)})

sns.boxplot(vc.ratio, ax=ax_box)
sns.distplot(vc.ratio, ax=ax_hist)
 
ax_box.set(xlabel='')

criteria = []

for i in range(13) : 
    criteria.append(vc.head(i+1).value.tolist())

import matplotlib.pyplot as plt

x = list(range(1,14))
y = pct
plt.plot(x, y)
plt.show()

good = criteria[-1]
trash = list(set(vc.value) - set(criteria[-1]))

Li = []

for i in range(len(aud)) :
    tmp_dict = dict(sorted(dict(zip(aud['long'][i], aud['short'][i])).items()))
    [tmp_dict.pop(key, None) for key in trash]
    Li.append(tmp_dict)

res = pd.DataFrame.from_records(Li, index = aud.device_ifa.values)

res.head()