text_plain_output_1.png
from keras.preprocessing.text import Tokenizer

import pandas as pd

data = pd.read_csv('../input/mondaq_data_adeel.csv', header=None)

columns = ['article_id', 'knt', 'unknt', 'her_knt', 'unher_knt', 'company_id', 'company_name', 'country_id', 'country_desc', 'primary_topic_id', 'topic_desc', 'article_start_date', 'daysold', 'topics', 'coauthors', 'linkedinphoto', 'title', 'body']

data.columns = columns

(train_df.shape, val_df.shape)

embed_size = 100
max_features = 100000
maxlen = 1400

train_X = train_df['entire_text'].fillna('##').values
val_X = val_df['entire_text'].fillna('##').values

tokenizer = Tokenizer(num_words=max_features)
tokenizer.fit_on_texts(list(data['entire_text'].fillna('##').values))
train_X = tokenizer.texts_to_sequences(train_X)
val_X = tokenizer.texts_to_sequences(val_X)
print('after tokenization')
print(len(train_X))
print(len(val_X))