text_plain_output_1.png image_output_1.png
from nltk import word_tokenize

from scipy.cluster.hierarchy import dendrogram, linkage

from scipy.cluster.hierarchy import fcluster

from sklearn.cluster import KMeans

from sklearn.manifold import TSNE

from unidecode import unidecode

import bs4

import nltk

import pandas as pd

import pandas as pd

import seaborn as sns

import seaborn as sns

import string

import tensorflow_hub as hub

import unicodedata

pd.options.display.max_colwidth = 255
df = pd.read_csv('../input/airline-sentiment/Tweets.csv')

module_url = 'https://tfhub.dev/google/universal-sentence-encoder/4'
model = hub.load(module_url)
def embed(input):
    return model(input)

import seaborn as sns

df_neg = df.query("airline_sentiment == 'negative'").head(1000).copy()
df_neu = df.query("airline_sentiment == 'neutral'").head(1000).copy()
df_pos = df.query("airline_sentiment == 'positive'").head(1000).copy()
df_balanced = pd.concat([df_neg, df_neu, df_pos], ignore_index=True)
df_balanced = df_balanced.sample(frac=1).reset_index(drop=True)
comments = df_balanced['text'].copy()

def limpa_dataframe(data, column, hmtl='s', emoji='s', punct='s', lower='s', stopw='s', lng='english', token='s', stm='s', lmz='s'):
    import nltk
    from nltk import word_tokenize
    from nltk.stem.porter import PorterStemmer
    from nltk.stem import WordNetLemmatizer
    import bs4
    import string
    wn = nltk.WordNetLemmatizer()
    ps = nltk.PorterStemmer()
    stopword = nltk.corpus.stopwords.words(lng)
    if hmtl == 's':
        data[column] = data[column].apply(lambda x: bs4.BeautifulSoup(x, 'lxml').get_text())
    def deEmojify(inputString):
        import unicodedata
        from unidecode import unidecode
        returnString = ''
        for character in inputString:
            try:
                character.encode('ascii')
                returnString += character
            except UnicodeEncodeError:
                replaced = unidecode(str(character))
                if replaced != '':
                    returnString += replaced
                else:
                    try:
                        returnString += '[' + unicodedata.name(character) + ']'
                    except ValueError:
                        returnString += '[x]'
        return returnString
    if emoji == 's':
        data[column] = data[column].apply(lambda x: deEmojify(x))
    def remove_punct(text):
        text_nopunct = ''.join([char for char in text if char not in string.punctuation])
        return text_nopunct
    if punct == 's':
        data[column] = data[column].apply(lambda x: remove_punct(x))
    if lower == 's':
        data[column] = [token.lower() for token in data[column]]
    if token == 's':
        data[column] = [word_tokenize(word) for word in data[column]]
    def remove_stopwords(tokenized_list):
        text = [word for word in tokenized_list if word not in stopword]
        return text
    if stopw == 's':
        data[column] = data[column].apply(lambda x: remove_stopwords(x))
    def stemming(tokenized_text):
        text = [ps.stem(word) for word in tokenized_text]
        return text
    if stm == 's':
        data[column] = data[column].apply(lambda x: stemming(x))
    def lemmatizing(tokenized_text):
        text = [wn.lemmatize(word) for word in tokenized_text]
        return text
    if lmz == 's':
        data[column] = data[column].apply(lambda x: lemmatizing(x))
    data[column] = [' '.join(word) for word in data[column]]
    return data

df_clean = df_balanced.copy()
df_clean = limpa_dataframe(df_clean, column='text', hmtl='s', emoji='s', punct='s', stopw='s', lng='english', token='s', stm='s', lmz='s')

messages = df_clean['text']
message_embeddings = embed(messages)

from sklearn.manifold import TSNE
tsne = TSNE(n_components=2, perplexity=5)
tsne_result = tsne.fit_transform(message_embeddings)
tsne_df = pd.DataFrame({'X': tsne_result[:, 0], 'Y': tsne_result[:, 1]})

from scipy.spatial.distance import pdist
from scipy.cluster.hierarchy import dendrogram, linkage
from scipy.cluster.hierarchy import cophenet
from scipy.cluster.hierarchy import fcluster
X = tsne_result
Z = linkage(X, 'average', 'cosine')
threshold = 1 - 0.9
c = fcluster(Z, threshold, criterion='distance')
myset = set(c.tolist())
my_list = list(set(myset))

from sklearn.cluster import KMeans
clf_k = KMeans(n_clusters=3, init='k-means++', max_iter=300, n_init=5, random_state=0)
pred_y_k = clf_k.fit_predict(X)
sns.scatterplot(x='X', y='Y', hue=pred_y_k, palette=['red', 'orange', 'blue'], data=tsne_df)