text_plain_output_3.png text_plain_output_2.png text_plain_output_1.png
from gingerit.gingerit import GingerIt

from tqdm.notebook import tqdm

from symspellpy.symspellpy import SymSpell, Verbosity
import pkg_resources
import re, string, json
import spacy
def normalization_pipeline(sentences):
    sentences = simplify_punctuation_and_whitespace(sentences)
    sentences = normalize_contractions(sentences)
    return sentences
def simplify_punctuation_and_whitespace(sentence_list):
    """
    using more than 4 ALLCAPS words will add EMPW and puntuation like !!!!! will get EMPP
    """
    norm_sents = []
    for sentence in tqdm(sentence_list):
        sent = _replace_urls(sentence)
        sent = _mention_hash(sent)
        sent = _simplify_punctuation(sent)
        sent = _reduce_repetitions(sent)
        sent = _normalize_whitespace(sent)
        norm_sents.append(sent)
    return norm_sents
def _replace_urls(text):
    url_regex = '(https?:\\/\\/(?:www\\.|(?!www))[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\\.[^\\s]{2,}|www\\.[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\\.[^\\s]{2,}|https?:\\/\\/(?:www\\.|(?!www))[a-zA-Z0-9]+\\.[^\\s]{2,}|www\\.[a-zA-Z0-9]+\\.[^\\s]{2,})'
    text = re.sub(url_regex, '-URL-', text)
    return text
def _mention_hash(in_str):
    """
    replacing @MENTION and #HASHTAG 
    BEWARE OF USES OF # AND @ AND SPACES BETWEEN THEM
    """
    in_str = str(in_str)
    in_str = re.sub('@', '@MEN ', in_str, flags=re.IGNORECASE)
    in_str = re.sub('#', '#HAS ', in_str, flags=re.IGNORECASE)
    return in_str.strip()
def _simplify_punctuation(text):
    """
    This function simplifies doubled or more complex punctuation. The exception is '...'. #?! ??? !!!
    """
    corrected = str(text)
    corrected = re.sub('([!?,;])\\1+', '\\1\\1 <-EMPP', corrected)
    corrected = re.sub('\\.{2,}', '...', corrected)
    return corrected
def _reduce_repetitions(text):
    """
    Auxiliary function to help with exxagerated words.
    Examples:
        woooooords -> woords
        yaaaaaaaaaaaaaaay -> yaay
        door -> dor
    """
    correction = str(text)
    for index, words in enumerate(str(text).split()):
        if _is_EMP_word(words) == True:
            correction = correction.replace(words, words + ' <-EMPW')
        if (len(words) > 4) & words.isupper() == True:
            correction = correction.replace(words, words + ' <-EMPU')
    return re.sub('([\\w])\\1+', '\\1\\1', correction)
def _is_EMP_word(word):
    """
    True/ False: checks if the word has 3 consecutive characters"""
    count = 1
    if len(word) > 1:
        for i in range(1, len(word)):
            if word[i] in string.punctuation:
                return False
            if word[i - 1] == word[i]:
                count += 1
                if count >= 3:
                    return True
            else:
                if count >= 3:
                    return True
                count = 1
    else:
        return False
    return False
def _normalize_whitespace(text):
    """
    This function normalizes whitespaces, removing duplicates.
    """
    corrected = str(text)
    corrected = re.sub('//t', '\\t', corrected)
    corrected = re.sub('( )\\1+', '\\1', corrected)
    corrected = re.sub('(\\n)\\1+', '\\1', corrected)
    corrected = re.sub('(\\r)\\1+', '\\1', corrected)
    corrected = re.sub('(\\t)\\1+', '\\1', corrected)
    return corrected.strip(' ')
def normalize_contractions(sentence_list):
    contraction_list = json.loads(open('/kaggle/input/english-contractions/english_contractions.json.txt', 'r').read())
    character_entity = {'&lt;3': 'heart', '&amp': 'and', '&quot;': ' quote '}
    contraction_list = {**contraction_list, **character_entity}
    norm_sents = []
    for sentence in tqdm(sentence_list):
        norm_sents.append(_normalize_contractions_slang_emoji_entity(sentence, contraction_list))
    return norm_sents
def _normalize_contractions_slang_emoji_entity(text, contractions):
    """
    part1:normalizes english contractions.
    """
    for word in text.split():
        if word.lower() in contractions:
            text = text.replace(word, contractions[word.lower()])
    '\n    part 2: using gingerit slang correction:\n    '
    parser = GingerIt()
    result = parser.parse(text)
    sentence = result['result']
    '\n    part3: emoji and character entity reference conversion to meaning\n    '
    emoticons = emot_obj.emoticons(sentence)
    printing = False
    for i in range(0, len(emoticons['value'])):
        sentence = sentence.replace(emoticons['value'][i], emoticons['mean'][i])
    return sentence

original_examples = ['hi @someone WATCH me #proud :) ;) ......  !!!!!    wanna go tHeRe bc my finls clooooose &quot;bananas&quot; &amp; ']
preprocessed_examples = normalization_pipeline(original_examples)
for example_index, example in enumerate(preprocessed_examples):
    print(original_examples[example_index])
    print(example)