text_plain_output_2.png text_plain_output_1.png
from nltk.corpus import stopwords

from nltk.stem import PorterStemmer

from nltk.stem import WordNetLemmatizer

from nltk.tokenize import word_tokenize, sent_tokenize

from sklearn.datasets import fetch_20newsgroups

import re

twenty_train = fetch_20newsgroups(subset='train', shuffle=True, random_state=42)

def clean_tag(text):
    return re.sub('<.*?>', '', text)
def clean_url(text):
    return re.sub('http\\S+', '', text)
def clean_special_character(text):
    return re.sub('[^a-zA-Z]', ' ', text)
def clean_uppercase(text):
    return str(text).lower()
def sent_tokenization(text):
    return sent_tokenize(text)
def tokenization(text):
    return word_tokenize(text)
def clean_stop_word(tokens):
    stop_words = set(stopwords.words('english'))
    return [token for token in tokens if token not in stop_words]
def steam(tokens):
    return [PorterStemmer().stem(token) for token in tokens]
def lenmatization(tokens):
    return [WordNetLemmatizer().lemmatize(word=token, pos='v') for token in tokens]
def clean_length(tokens):
    return [token for token in tokens if len(token) > 2]
def convert_2_string(text):
    return ' '.join(text)

def clean(text):
    res = clean_url(text)
    res = clean_special_character(res)
    res = clean_uppercase(res)
    res = tokenization(res)
    res = clean_stop_word(res)
    res = lenmatization(res)
    res = clean_length(res)
    return convert_2_string(res)

example = twenty_train.data[0]
after_clean = clean(example)
print(example, after_clean)