text_plain_output_1.png
from typing import List

import contractions

import ftfy

import re

import spacy

import string

import spacy
import scispacy
nlp = spacy.load('../input/scispacymodels/en_core_sci_sm/en_core_sci_sm-0.2.4')
nlp.max_length = 2000000

import re
CURRENCIES = {'$': 'USD', 'zł': 'PLN', '£': 'GBP', '¥': 'JPY', '฿': 'THB', '₡': 'CRC', '₦': 'NGN', '₩': 'KRW', '₪': 'ILS', '₫': 'VND', '€': 'EUR', '₱': 'PHP', '₲': 'PYG', '₴': 'UAH', '₹': 'INR'}
RE_NUMBER = re.compile('(?:^|(?<=[^\\w,.]))[+–-]?(([1-9]\\d{0,2}(,\\d{3})+(\\.\\d*)?)|([1-9]\\d{0,2}([ .]\\d{3})+(,\\d*)?)|(\\d*?[.,]\\d+)|\\d+)(?:$|(?=\\b))')
RE_URL = re.compile('((http://www\\.|https://www\\.|http://|https://)?' + '[a-z0-9]+([\\-.][a-z0-9]+)*\\.[a-z]{2,5}(:[0-9]{1,5})?(/.*)?)')
STOP_WORDS = {'a', 'an', 'and', 'are', 'as', 'at', 'be', 'but', 'by', 'for', 'if', 'in', 'into', 'is', 'it', 'no', 'not', 'of', 'on', 'or', 'such', 'that', 'the', 'their', 'then', 'there', 'these', 'they', 'this', 'to', 'was', 'will', 'with'}

import string
from typing import List
import ftfy
import contractions
def clean_tokenized_sentence(tokens: List[str], unicode_normalization='NFC', unpack_contractions=False, replace_currency_symbols=False, remove_punct=True, remove_numbers=False, lowercase=True, remove_urls=True, remove_stop_words=True) -> str:
    if remove_stop_words:
        tokens = [token for token in tokens if token not in STOP_WORDS]
    sentence = ' '.join(tokens)
    if unicode_normalization:
        sentence = ftfy.fix_text(sentence, normalization=unicode_normalization)
    if unpack_contractions:
        sentence = contractions.fix(sentence, slang=False)
    if replace_currency_symbols:
        for currency_sign, currency_tok in CURRENCIES.items():
            sentence = sentence.replace(currency_sign, f'{currency_tok} ')
    if remove_urls:
        sentence = RE_URL.sub('_URL_', sentence)
    if remove_punct:
        sentence = sentence.translate(str.maketrans('', '', string.punctuation))
    sentence = re.sub(' +', ' ', sentence)
    if remove_numbers:
        sentence = RE_NUMBER.sub('_NUMBER_', sentence)
    if lowercase:
        sentence = sentence.lower()
    return sentence

def clean_sentence(sentence) -> str:
    doc = nlp(sentence)
    tokens = [str(token) for token in doc]
    return clean_tokenized_sentence(tokens)

print(clean_sentence("Let's clean this sentence!"))