application_vnd.jupyter.stderr_output_1.png
from collections import Counter

from gingerit.gingerit import GingerIt

from sklearn.model_selection import train_test_split

from tqdm.notebook import tqdm

import matplotlib.pyplot as plt

import os

import pandas as pd  # data processing, CSV file I/O (e.g. pd.read_csv)

import os
df = pd.read_csv('/kaggle/input/emotion-detection-from-text/tweet_emotions.csv')

df.content.iloc[-10:]

df.sentiment.value_counts()

df = df[df.sentiment != 'anger']
df = df[df.sentiment != 'boredom']
df = df[df.sentiment != 'enthusiasm']
df = df[df.sentiment != 'empty']
df = df[df.sentiment != 'sentiment']

df.sentiment.value_counts()

possible_labels = df.sentiment.unique()

label_dict = {}
for index, possible_label in enumerate(possible_labels):
    label_dict[possible_label] = index

df['label'] = df.sentiment.replace(label_dict)

X_train, X_val, y_train, y_val = train_test_split(df.index.values, df.label.values, test_size=0.15, stratify=df.label.values)

df['data_type'] = ['not_set'] * df.shape[0]

df.loc[X_train, 'data_type'] = 'train'
df.loc[X_val, 'data_type'] = 'val'

df.groupby(['sentiment', 'label', 'data_type']).count()

from symspellpy.symspellpy import SymSpell, Verbosity
import pkg_resources
import re, string, json
import spacy
def normalization_pipeline(sentences):
    sentences = simplify_punctuation_and_whitespace(sentences)
    sentences = normalize_contractions(sentences)
    return sentences
def simplify_punctuation_and_whitespace(sentence_list):
    """
    using more than 4 ALLCAPS words will add EMPW and puntuation like !!!!! will get EMPP
    """
    norm_sents = []
    for sentence in tqdm(sentence_list):
        sent = _replace_urls(sentence)
        sent = _mention_hash(sent)
        sent = _simplify_punctuation(sent)
        sent = _reduce_repetitions(sent)
        sent = _normalize_whitespace(sent)
        norm_sents.append(sent)
    return norm_sents
def _replace_urls(text):
    url_regex = '(https?:\\/\\/(?:www\\.|(?!www))[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\\.[^\\s]{2,}|www\\.[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\\.[^\\s]{2,}|https?:\\/\\/(?:www\\.|(?!www))[a-zA-Z0-9]+\\.[^\\s]{2,}|www\\.[a-zA-Z0-9]+\\.[^\\s]{2,})'
    text = re.sub(url_regex, '-URL-', text)
    return text
def _mention_hash(in_str):
    """
    replacing @MENTION and #HASHTAG 
    BEWARE OF USES OF # AND @ AND SPACES BETWEEN THEM
    """
    in_str = str(in_str)
    in_str = re.sub('@', '@MEN ', in_str, flags=re.IGNORECASE)
    in_str = re.sub('#', '#HAS ', in_str, flags=re.IGNORECASE)
    return in_str.strip()
def _simplify_punctuation(text):
    """
    This function simplifies doubled or more complex punctuation. The exception is '...'. #?! ??? !!!
    """
    corrected = str(text)
    corrected = re.sub('([!?,;])\\1+', '\\1\\1 <-EMPP', corrected)
    corrected = re.sub('\\.{2,}', '...', corrected)
    return corrected
def _reduce_repetitions(text):
    """
    Auxiliary function to help with exxagerated words.
    Examples:
        woooooords -> woords
        yaaaaaaaaaaaaaaay -> yaay
        door -> dor
    """
    correction = str(text)
    for index, words in enumerate(str(text).split()):
        if _is_EMP_word(words) == True:
            correction = correction.replace(words, words + ' <-EMPW')
        if (len(words) > 4) & words.isupper() == True:
            correction = correction.replace(words, words + ' <-EMPU')
    return re.sub('([\\w])\\1+', '\\1\\1', correction)
def _is_EMP_word(word):
    """
    True/ False: checks if the word has 3 consecutive characters"""
    count = 1
    if len(word) > 1:
        for i in range(1, len(word)):
            if word[i] in string.punctuation:
                return False
            if word[i - 1] == word[i]:
                count += 1
                if count >= 3:
                    return True
            else:
                if count >= 3:
                    return True
                count = 1
    else:
        return False
    return False
def _normalize_whitespace(text):
    """
    This function normalizes whitespaces, removing duplicates.
    """
    corrected = str(text)
    corrected = re.sub('//t', '\\t', corrected)
    corrected = re.sub('( )\\1+', '\\1', corrected)
    corrected = re.sub('(\\n)\\1+', '\\1', corrected)
    corrected = re.sub('(\\r)\\1+', '\\1', corrected)
    corrected = re.sub('(\\t)\\1+', '\\1', corrected)
    return corrected.strip(' ')
def normalize_contractions(sentence_list):
    contraction_list = json.loads(open('/kaggle/input/english-contractions/english_contractions.json.txt', 'r').read())
    character_entity = {'&lt;3': 'heart', '&amp': 'and', '&quot;': ' quote '}
    contraction_list = {**contraction_list, **character_entity}
    norm_sents = []
    for sentence in tqdm(sentence_list):
        norm_sents.append(_normalize_contractions_slang_emoji_entity(sentence, contraction_list))
    return norm_sents
def _normalize_contractions_slang_emoji_entity(text, contractions):
    """
    part1:normalizes english contractions.
    """
    for word in text.split():
        if word.lower() in contractions:
            text = text.replace(word, contractions[word.lower()])
    '\n    part 2: using gingerit slang correction:\n    '
    parser = GingerIt()
    result = parser.parse(text)
    sentence = result['result']
    '\n    part3: emoji and character entity reference conversion to meaning\n    '
    emoticons = emot_obj.emoticons(sentence)
    printing = False
    for i in range(0, len(emoticons['value'])):
        sentence = sentence.replace(emoticons['value'][i], emoticons['mean'][i])
    return sentence

import matplotlib.pyplot as plt
from collections import Counter
tokenizee = []
for words in tqdm(range(1, len(df.content) - 1)):
    tokenizee.append(spacy_process(df.content[words]))
words = Counter()
for s in tokenizee:
    for w in s:
        words[w] += 1
sorted_words = list(words.keys())
sorted_words.sort(key=lambda w: words[w], reverse=True)
print(f'Number of different Tokens in our Dataset: {len(sorted_words)}')
print(sorted_words[:100])
count_occurences = sum(words.values())
accumulated = 0
counter = 0
while accumulated < count_occurences * 0.8:
    accumulated += words[sorted_words[counter]]
    counter += 1
print(f'The {counter * 100 / len(words)}% most common words account for the {accumulated * 100 / count_occurences}% of the occurrences')
plt.bar(range(100), [words[w] for w in sorted_words[:100]])
plt.show()