text_plain_output_1.png
from keras.preprocessing import sequence

from keras.preprocessing.text import Tokenizer

import numpy as np

import pandas as pd

train_data = pd.read_csv('../input/train.csv')
train_data.sample(5)

train_data['target'].value_counts()[0] / train_data.shape[0] * 100

train_data.isna().sum()

train_data.drop(['id'], axis=1, inplace=True)

X = train_data['comment_text']
y = train_data['target']

max_words = 10000
tok = Tokenizer(num_words=max_words)
tok.fit_on_texts(X)

aux_columns = ['capitals', 'exclamation_points', 'total_length']

train_meta_features = np.asarray(x_train[aux_columns])
test_meta_features = np.asarray(x_test[aux_columns])

x_train = x_train['comment_text']
x_test = x_test['comment_text']

max_words = 10000
tok = Tokenizer(num_words=max_words)
tok.fit_on_texts(x_train)
sequences = tok.texts_to_sequences(x_train)
sequences_matrix = sequence.pad_sequences(sequences)

print(sequences_matrix.shape)
print(train_meta_features.shape)