text_plain_output_2.png application_vnd.jupyter.stderr_output_1.png
from nltk.corpus import stopwords

from sklearn.feature_extraction.text import HashingVectorizer

from sklearn.linear_model import SGDClassifier

import datetime

import numpy as np

import re

def calc_len_partial(X_train, limit=15):
    i = 1
    partial_len = len(X_train)
    div_len = 0
    while i:
        if partial_len % 2:
            partial_len = len(X_train) / 2
            div_len += 2
        elif partial_len % 3:
            partial_len = len(X_train) / 3
            div_len += 3
        elif partial_len % 5:
            partial_len = len(X_train) / 5
            div_len += 5
        elif partial_len % 7:
            partial_len = len(X_train) / 7
            div_len += 7
        elif partial_len % 11:
            partial_len = len(X_train) / 11
            div_len += 11
        else:
            break
        if div_len > limit:
            break
    return len(X_train) / div_len

def preprocessor(text):
    text = str(text)
    text = re.sub('<[^>]*>', '', text)
    text = re.sub('[\\W]+', ' ', text.lower())
    text = text.rstrip().lstrip()
    return text

stop = stopwords.words('english')
vect = HashingVectorizer(decode_error='ignore', n_features=2 ** 21, preprocessor=preprocessor, stop_words=stop, ngram_range=(1, 3))
clf = SGDClassifier(loss='log', random_state=1, n_iter=1)

print(datetime.datetime.now())
len_partial = int(calc_len_partial(X_train=X_train, limit=20))
classes = np.array([0, 1, 2])
for i in range(round(len(X_train) / len_partial)):
    X_train_ml = X_train[i:len_partial * (i + 1)]
    y_train_ml = y_train[i:len_partial * (i + 1)]
    X_train_ml = vect.transform(X_train_ml)
    clf.partial_fit(X_train_ml, y_train_ml, classes=classes)
print(datetime.datetime.now())