image_output_1.png
from keras.preprocessing.sequence import pad_sequences

from keras.preprocessing.text import Tokenizer

from keras.preprocessing.text import Tokenizer

from sklearn.model_selection import train_test_split

import pandas as pd

import re

import seaborn as sns

data_train = pd.read_csv('../input/nlp-getting-started/train.csv')
data_test = pd.read_csv('../input/nlp-getting-started/test.csv')

data_train = data_train[['id', 'text', 'target']]
data_test = data_test[['id', 'text']]

s = data_train.target.value_counts()

def clean_text(data):
    data['clean_text'] = data['text'].str.lower()
    data['clean_text'] = data['clean_text'].apply(lambda elem: re.sub('http\\S+', '', elem))
    data['clean_text'] = data['clean_text'].apply(lambda elem: re.sub('[^\\w\\s]', '', elem))
    data['clean_text'] = data['clean_text'].apply(lambda elem: re.sub('/n', '', elem))
    data['clean_text'] = data['clean_text'].apply(lambda elem: re.sub('\\d+', '', elem))
    data['clean_text'] = data['clean_text'].apply(lambda elem: re.sub('\\s+', ' ', elem))
    data['clean_text'] = data['clean_text'].apply(lambda elem: re.sub('\\s+[a-zA-Z]\\s+', ' ', elem))
    return data

data_train = clean_text(data_train)
data_test = clean_text(data_test)

max_fatures = 5000
tokenizer = Tokenizer(num_words=max_fatures, split=' ')
tokenizer.fit_on_texts(data_train['clean_text'].values)
X = tokenizer.texts_to_sequences(data_train['clean_text'].values)
X = pad_sequences(X, maxlen=31, padding='post')

Y = data_train['target'].values
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.1, random_state=0)
print(X_train.shape, Y_train.shape)
print(X_test.shape, Y_test.shape)