text_html_output_1.png
from scipy.spatial.distance import pdist

import nltk

import numpy as np

import numpy as np

import pandas as pd

import pandas as pd

import re

import tensorflow_hub as hub

tweets = pd.read_csv('/kaggle/input/data-trab-2/Tweets.csv', error_bad_lines=False)
tweets

doc = tweets['text']
doc

wpt = nltk.WordPunctTokenizer()
stop_words = nltk.corpus.stopwords.words('english')
def normalize_document(doc):
    doc = re.sub('[^a-zA-Z\\s]', '', doc)
    doc = doc.lower()
    doc = doc.strip()
    tokens = wpt.tokenize(doc)
    filtered_tokens = [token for token in tokens if token not in stop_words]
    doc = ' '.join(filtered_tokens)
    return doc
normalize_doc = np.vectorize(normalize_document)
norm_doc = normalize_doc(doc)
norm_doc

module_url = 'https://tfhub.dev/google/universal-sentence-encoder/4'
model = hub.load(module_url)
def embed(input):
    return model(input)

X = embed(norm_doc)

Y = pdist(X, 'cosine')