text_plain_output_1.png
from sklearn.feature_extraction.text import TfidfVectorizer

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

df = pd.read_csv('../input/machathon-20-final-round/train_ara.csv')

classes = df.intent.unique()
classes

df['intent_id'] = df['intent'].factorize()[0]
intent_id_df = df[['intent', 'intent_id']].drop_duplicates()
intent_to_id = dict(intent_id_df.values)
id_to_intent = dict(intent_id_df[['intent_id', 'intent']].values)

tfidf = TfidfVectorizer(sublinear_tf=True, ngram_range=(1, 2))
features = tfidf.fit_transform(df['lem']).toarray()
labels = df.intent_id
print('Each of the %d intents is represented by %d features (TF-IDF score of unigrams and bigrams)' % features.shape)