text_plain_output_1.png
from scipy.cluster.hierarchy import ward, fcluster, linkage

from scipy.spatial.distance import pdist

import nltk

import numpy as np

import numpy as np

import pandas as pd

import pandas as pd

import plotly.figure_factory as ff

import re

import tensorflow_hub as hub

tweets = pd.read_csv('/kaggle/input/data-trab-2/Tweets.csv', error_bad_lines=False)
tweets

doc = tweets['text']
doc

wpt = nltk.WordPunctTokenizer()
stop_words = nltk.corpus.stopwords.words('english')
def normalize_document(doc):
    doc = re.sub('[^a-zA-Z\\s]', '', doc)
    doc = doc.lower()
    doc = doc.strip()
    tokens = wpt.tokenize(doc)
    filtered_tokens = [token for token in tokens if token not in stop_words]
    doc = ' '.join(filtered_tokens)
    return doc
normalize_doc = np.vectorize(normalize_document)
norm_doc = normalize_doc(doc)
norm_doc

module_url = 'https://tfhub.dev/google/universal-sentence-encoder/4'
model = hub.load(module_url)
def embed(input):
    return model(input)

X = embed(norm_doc)

Y = pdist(X, 'cosine')

Z = linkage(y=Y)

ff.create_dendrogram(Z)