text_plain_output_1.png
from nltk import word_tokenize

from unidecode import unidecode

import bs4

import nltk

import pandas as pd

import pandas as pd

import seaborn as sns

import seaborn as sns

import string

import unicodedata

pd.options.display.max_colwidth = 255
df = pd.read_csv('../input/airline-sentiment/Tweets.csv')

import seaborn as sns

df_neg = df.query("airline_sentiment == 'negative'").head(1000).copy()
df_neu = df.query("airline_sentiment == 'neutral'").head(1000).copy()
df_pos = df.query("airline_sentiment == 'positive'").head(1000).copy()
df_balanced = pd.concat([df_neg, df_neu, df_pos], ignore_index=True)
df_balanced = df_balanced.sample(frac=1).reset_index(drop=True)
comments = df_balanced['text'].copy()

def limpa_dataframe(data, column, hmtl='s', emoji='s', punct='s', lower='s', stopw='s', lng='english', token='s', stm='s', lmz='s'):
    import nltk
    from nltk import word_tokenize
    from nltk.stem.porter import PorterStemmer
    from nltk.stem import WordNetLemmatizer
    import bs4
    import string
    wn = nltk.WordNetLemmatizer()
    ps = nltk.PorterStemmer()
    stopword = nltk.corpus.stopwords.words(lng)
    if hmtl == 's':
        data[column] = data[column].apply(lambda x: bs4.BeautifulSoup(x, 'lxml').get_text())
    def deEmojify(inputString):
        import unicodedata
        from unidecode import unidecode
        returnString = ''
        for character in inputString:
            try:
                character.encode('ascii')
                returnString += character
            except UnicodeEncodeError:
                replaced = unidecode(str(character))
                if replaced != '':
                    returnString += replaced
                else:
                    try:
                        returnString += '[' + unicodedata.name(character) + ']'
                    except ValueError:
                        returnString += '[x]'
        return returnString
    if emoji == 's':
        data[column] = data[column].apply(lambda x: deEmojify(x))
    def remove_punct(text):
        text_nopunct = ''.join([char for char in text if char not in string.punctuation])
        return text_nopunct
    if punct == 's':
        data[column] = data[column].apply(lambda x: remove_punct(x))
    if lower == 's':
        data[column] = [token.lower() for token in data[column]]
    if token == 's':
        data[column] = [word_tokenize(word) for word in data[column]]
    def remove_stopwords(tokenized_list):
        text = [word for word in tokenized_list if word not in stopword]
        return text
    if stopw == 's':
        data[column] = data[column].apply(lambda x: remove_stopwords(x))
    def stemming(tokenized_text):
        text = [ps.stem(word) for word in tokenized_text]
        return text
    if stm == 's':
        data[column] = data[column].apply(lambda x: stemming(x))
    def lemmatizing(tokenized_text):
        text = [wn.lemmatize(word) for word in tokenized_text]
        return text
    if lmz == 's':
        data[column] = data[column].apply(lambda x: lemmatizing(x))
    data[column] = [' '.join(word) for word in data[column]]
    return data

df_clean = df_balanced.copy()
df_clean = limpa_dataframe(df_clean, column='text', hmtl='s', emoji='s', punct='s', stopw='s', lng='english', token='s', stm='s', lmz='s')
df_clean['text'].head(30)