text_plain_output_1.png
from keras.preprocessing.sequence import pad_sequences

from keras.preprocessing.text import Tokenizer

tokenizer = Tokenizer()
tokenizer.fit_on_texts(X_train)
word_index = tokenizer.word_index
X_train_tok = tokenizer.texts_to_sequences(X_train)
X_test_tok = tokenizer.texts_to_sequences(X_test)

vocab_size = len(tokenizer.word_index) + 1
X_train_tok = pad_sequences(X_train_tok, padding='post', maxlen=30)
X_test_tok = pad_sequences(X_test_tok, padding='post', maxlen=30)

print('X_train shape: ', X_train_tok.shape)
print('X_test shape: ', X_test_tok.shape)