text_plain_output_1.png
from nltk.stem.wordnet import WordNetLemmatizer

from sklearn.feature_extraction.text import CountVectorizer

from sklearn.linear_model import LogisticRegression

from sklearn.metrics import accuracy_score

import matplotlib.pyplot as plt

import pandas as pd

import re

import pandas as pd
df = pd.read_csv('../input/bbc-text.csv')

import re
from nltk.stem.wordnet import WordNetLemmatizer
stop_words = ['in', 'of', 'at', 'a', 'the']
def pre_process(text):
    text = str(text).lower()
    text = re.sub('((\\d+)[\\.])', '', text)
    text = re.sub('&lt;/?.*?&gt;', ' &lt;&gt; ', text)
    text = text.replace('dont', "don't")
    text = re.sub("[^a-zA-Z0-9.']+", ' ', text)
    "\n    Don't include this in the beginning. \n    First check if there are some patterns that may be lost if we remove stopwords.\n    "
    text = [word for word in text.split(' ') if word not in stop_words]
    lmtzr = WordNetLemmatizer()
    text = ' '.join((lmtzr.lemmatize(i) for i in text))
    return text

for i in range(len(df)):
    df.text[i] = pre_process(df.text[i])

# Visualize the distribution of categories
import matplotlib.pyplot as plt
fig = plt.figure(figsize=(10,6))
df.groupby('category').text.count().plot.bar(ylim=0)
plt.show()

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score
train_data = df.text[0:int(0.75 * len(df))]
test_data = df.text[int(0.75 * len(df)) + 1:]
train_target = df.category[0:int(0.75 * len(df))]
test_target = df.category[int(0.75 * len(df)) + 1:]
stop_words = ['in', 'of', 'at', 'a', 'the']
ngram_vectorizer = CountVectorizer(binary=True, ngram_range=(1, 3), stop_words=stop_words)
ngram_vectorizer.fit(train_data)
X_train = ngram_vectorizer.transform(train_data)
X_test = ngram_vectorizer.transform(test_data)

model = LogisticRegression()
model.fit(X_train, train_target)
test_acc = accuracy_score(test_target, model.predict(X_test))
print('Test accuracy: {0:.2f}%'.format(100 * test_acc))