text_plain_output_1.png
from tensorflow.keras.layers.experimental.preprocessing import TextVectorization

import numpy as np # linear algebra

import random

sent_lens = [len(sentence.split()) for sentence in train_sentences]
sent_lens[:10]

avg_len = np.mean(sent_lens)
avg_len

out_len_seq = np.percentile(sent_lens, 95)
out_len_seq

max_tokens = 65000
output_sequence_length = int(out_len_seq)

text_vectorizer = TextVectorization(max_tokens=max_tokens, output_sequence_length=output_sequence_length)

text_vectorizer.adapt(train_sentences)

random_sen = random.choice(train_sentences)
random_sen

text_vectorizer([random_sen])