text_plain_output_1.png
from sklearn.feature_extraction.text import TfidfVectorizer

from sklearn.feature_selection import chi2

import numpy as np # linear algebra

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

df = pd.read_csv('../input/machathon-20-final-round/train_ara.csv')

classes = df.intent.unique()
classes

df['intent_id'] = df['intent'].factorize()[0]
intent_id_df = df[['intent', 'intent_id']].drop_duplicates()
intent_to_id = dict(intent_id_df.values)
id_to_intent = dict(intent_id_df[['intent_id', 'intent']].values)

tfidf = TfidfVectorizer(sublinear_tf=True, ngram_range=(1, 2))
features = tfidf.fit_transform(df['lem']).toarray()
labels = df.intent_id

N = 3
for intent, intent_id in sorted(intent_to_id.items()):
    features_chi2 = chi2(features, labels == intent_id)
    indices = np.argsort(features_chi2[0])
    feature_names = np.array(tfidf.get_feature_names())[indices]
    unigrams = [v for v in feature_names if len(v.split(' ')) == 1]
    bigrams = [v for v in feature_names if len(v.split(' ')) == 2]
    print('\n==> %s:' % intent)
    print('  * Most Correlated Unigrams are: %s' % ', '.join(unigrams[-N:]))
    print('  * Most Correlated Bigrams are: %s' % ', '.join(bigrams[-N:]))