text_html_output_1.png
from keras.preprocessing import sequence

from keras.preprocessing.text import Tokenizer

import pandas as pd

datasets_dir = ''
vnrows = None
datasets_dir = '../input/'
df = pd.read_csv(datasets_dir + 'fakenews_preprocessed_35k.csv', nrows=vnrows, encoding='utf-8')

df = df.loc[:, ~df.columns.str.contains('^Unnamed')]

df = df.sample(frac=1).reset_index(drop=True)

max_seq_len = 458

MAX_NB_WORDS = round(int(df.doc_length.sum() / max_seq_len))

tokenizer = Tokenizer(num_words=MAX_NB_WORDS, lower=True, char_level=False)

tokenizer.fit_on_texts(X_train)
word_index = tokenizer.word_index

word_seq_train = tokenizer.texts_to_sequences(X_train)
word_seq_train = sequence.pad_sequences(word_seq_train, maxlen=max_seq_len)

word_seq_train.shape