text_plain_output_1.png
from tensorflow.keras import layers

from tensorflow.keras.layers.experimental.preprocessing import TextVectorization

import numpy as np # linear algebra

import random

sent_lens = [len(sentence.split()) for sentence in train_sentences]
sent_lens[:10]

avg_len = np.mean(sent_lens)
avg_len

out_len_seq = np.percentile(sent_lens, 95)
out_len_seq

max_tokens = 65000
output_sequence_length = int(out_len_seq)

text_vectorizer = TextVectorization(max_tokens=max_tokens, output_sequence_length=output_sequence_length)

text_vectorizer.adapt(train_sentences)

random_sen = random.choice(train_sentences)
random_sen

news_vocab = text_vectorizer.get_vocabulary()
(print(f'Number of words in vocabulary: {len(news_vocab)}'),)

embedding = layers.Embedding(input_dim=max_tokens, output_dim=128, input_length=out_len_seq, embeddings_initializer='uniform')
embedding

print(embedding(text_vectorizer([random_sen])))