application_vnd.jupyter.stderr_output_3.png text_plain_output_2.png application_vnd.jupyter.stderr_output_1.png
from nltk.corpus import stopwords

from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer

from sklearn.model_selection import train_test_split

import gc

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

data = pd.read_csv('../input/train.csv')
y = data.deal_probability.copy()
X_train, X_test, y_train, y_test = train_test_split(data, y, test_size=0.2, random_state=23)
del data
gc.collect()

count_vectorizer_title = CountVectorizer(stop_words=stopwords.words('russian'), lowercase=True, ngram_range=(1, 2), max_features=8000)
title_counts = count_vectorizer_title.fit_transform(X_train['title'].append(X_test['title']))
train_title_counts = title_counts[:len(X_train)]
test_title_counts = title_counts[len(X_train):]
count_vectorizer_desc = TfidfVectorizer(stop_words=stopwords.words('russian'), lowercase=True, ngram_range=(1, 2), max_features=17000)
desc_counts = count_vectorizer_desc.fit_transform(X_train['description'].append(X_test['description']))
train_desc_counts = desc_counts[:len(X_train)]
test_desc_counts = desc_counts[len(X_train):]
(train_title_counts.shape, train_desc_counts.shape)