text_plain_output_1.png image_output_1.png
from nltk.corpus import stopwords

from nltk.stem import PorterStemmer

from nltk.stem import WordNetLemmatizer

from nltk.tokenize import word_tokenize, sent_tokenize

from sklearn import metrics

from sklearn import metrics

from sklearn.datasets import fetch_20newsgroups

from sklearn.feature_extraction.text import CountVectorizer

from sklearn.feature_extraction.text import TfidfTransformer

from sklearn.pipeline import Pipeline

from sklearn.svm import LinearSVC

from time import time

import matplotlib.pyplot as plt

import numpy as np

import pandas as pd

import re

import seaborn as sns

twenty_train = fetch_20newsgroups(subset='train', shuffle=True, random_state=42)

twenty_test = fetch_20newsgroups(subset='test')

def clean_tag(text):
    return re.sub('<.*?>', '', text)
def clean_url(text):
    return re.sub('http\\S+', '', text)
def clean_special_character(text):
    return re.sub('[^a-zA-Z]', ' ', text)
def clean_uppercase(text):
    return str(text).lower()
def sent_tokenization(text):
    return sent_tokenize(text)
def tokenization(text):
    return word_tokenize(text)
def clean_stop_word(tokens):
    stop_words = set(stopwords.words('english'))
    return [token for token in tokens if token not in stop_words]
def steam(tokens):
    return [PorterStemmer().stem(token) for token in tokens]
def lenmatization(tokens):
    return [WordNetLemmatizer().lemmatize(word=token, pos='v') for token in tokens]
def clean_length(tokens):
    return [token for token in tokens if len(token) > 2]
def convert_2_string(text):
    return ' '.join(text)

def clean(text):
    res = clean_url(text)
    res = clean_special_character(res)
    res = clean_uppercase(res)
    res = tokenization(res)
    res = clean_stop_word(res)
    res = lenmatization(res)
    res = clean_length(res)
    return convert_2_string(res)

example = twenty_train.data[0]
after_clean = clean(example)

processed_train_data = [clean(letter) for letter in twenty_train.data]
processed_test_data = [clean(letter) for letter in twenty_test.data]

reports = []

def draw_confusion_matrix(target, predicted, target_names=twenty_test.target_names, normalize=None):
    cm = metrics.confusion_matrix(target, predicted, normalize=normalize)
    df_cm = pd.DataFrame(cm, index=[i for i in target_names], columns=target_names)

def benchmark(pipeline, clf_name, X_train=processed_train_data, y_train=twenty_train.target, X_test=processed_test_data, y_test=twenty_test.target):
    report = []
    report.append(clf_name)
    t0 = time()
    pipeline.fit(X_train, y_train)
    train_time = time() - t0
    report.append(train_time)
    t0 = time()
    pred = pipeline.predict(X_test)
    test_time = time() - t0
    report.append(test_time)
    accuracy = metrics.accuracy_score(y_test, pred)
    report.append(accuracy)
    precision = metrics.precision_score(y_test, pred, average='micro')
    report.append(precision)
    recall = metrics.precision_score(y_test, pred, average='micro')
    report.append(recall)
    f1_score = metrics.f1_score(y_test, pred, average='micro')
    report.append(f1_score)
    mathew = metrics.matthews_corrcoef(y_test, pred)
    report.append(mathew)
    reports.append(report)
    clf = pipeline.named_steps.clf
    vectorizer = pipeline.named_steps.vect
    feature_names = vectorizer.get_feature_names()
    if hasattr(clf, 'coef_'):
        for i, label in enumerate(twenty_train.target_names):
            top10 = np.argsort(clf.coef_[i])[-10:]

linearSVC_pip = Pipeline([('vect', CountVectorizer(max_df=0.75, ngram_range=(1, 2))), ('tf', TfidfTransformer()), ('clf', LinearSVC(C=10))])

benchmark(linearSVC_pip, 'LinearSVC')