text_plain_output_1.png
from sklearn.feature_extraction.text import TfidfVectorizer

from tqdm import tqdm

import numpy as np

import pandas as pd

import seaborn as sns

train = pd.read_csv('../input/feedback-prize-english-language-learning/train.csv')
test = pd.read_csv('../input/feedback-prize-english-language-learning/test.csv')
ss = pd.read_csv('../input/feedback-prize-english-language-learning/sample_submission.csv')

def get_length_of_text(x):
    return len(x)

train.full_text.apply(lambda x: get_length_of_text(x)).hist()

train.isna().sum()

colormap = sns.color_palette('Blues')

df = train.copy()
target_vars = ['cohesion', 'syntax', 'vocabulary', 'phraseology', 'grammar', 'conventions']

from sklearn.feature_extraction.text import TfidfVectorizer
vectorizer = TfidfVectorizer(smooth_idf=True, sublinear_tf=True, max_features=5000)
vectorizer.fit(raw_documents=train.full_text)

def extract_vectors(x):
    vecs = vectorizer.transform(x)
    return vecs.toarray().flatten()
df['vecs'] = train.full_text.apply(lambda x: extract_vectors([x]))

feature_set = []
for i, row in tqdm(df.iterrows(), total=len(df)):
    vecs = row['vecs']
    vals = row[target_vars].astype(float)
    features = np.hstack([vecs, vals]).flatten()
    feature_set.append(features)
feature_set = np.array(feature_set)