text_plain_output_1.png
from imblearn.over_sampling import SMOTE

from nltk.corpus import stopwords

from nltk.stem import WordNetLemmatizer

from sklearn.feature_extraction.text import TfidfVectorizer

import matplotlib.pyplot as plt

import numpy as np # linear algebra

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import re
import seaborn as sns
from imblearn.over_sampling import SMOTE
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
stop_words = set(stopwords.words('english'))
from sklearn.preprocessing import LabelEncoder
from xgboost import XGBClassifier
from nltk.stem import WordNetLemmatizer
lemmatizer = WordNetLemmatizer()
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.svm import LinearSVC
from sklearn.model_selection import cross_val_score, train_test_split
from sklearn.metrics import confusion_matrix, accuracy_score

df_train = pd.read_excel('../input/Data_Train.xlsx')
df_test = pd.read_excel('../input/Data_Test.xlsx')

df_train.sample(5)

df_train.isna().sum()

labels = list(set(df_train['SECTION']))
counts = []
for label in labels:
    counts.append(np.count_nonzero(df_train['SECTION'] == label))

X = df_train['STORY']
y = df_train['SECTION']

tfidf_vec = TfidfVectorizer(ngram_range=(1, 2), stop_words=stop_words)
tfidf_vec.fit(X)
x_vec = tfidf_vec.transform(X)

sm = SMOTE(random_state=42)
x_vec, y = sm.fit_sample(x_vec, y)
labels = list(set(y))
counts = []
for label in labels:
    counts.append(np.count_nonzero(y == label))
plt.pie(counts, labels=labels, autopct='%1.1f%%')
plt.show()