text_plain_output_1.png
from gensim.models import Phrases

from gensim.models.phrases import Phraser

from nltk.corpus import stopwords

import pandas as pd

import pickle

import pickle

import re

import pandas as pd
import re
from nltk.corpus import stopwords
from gensim.models import word2vec
import pickle
import nltk.data
import os
tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')

path = '../input/'
TRAIN_DATA_FILE = f'{path}train.csv'
TEST_DATA_FILE = f'{path}test.csv'
train = pd.read_csv(TRAIN_DATA_FILE, header=0)
test = pd.read_csv(TEST_DATA_FILE, header=0)
all_comments = train['comment_text'].fillna('_na_').tolist() + test['comment_text'].fillna('_na_').tolist()
with open('all_comments.csv', 'w+') as comments_file:
    i = 0
    for comment in all_comments:
        comment = re.sub('[^a-zA-Z]', ' ', str(comment))
        comments_file.write('%s\n' % comment)

class FileToComments(object):
    def __init__(self, filename):
        self.filename = filename
        self.stop = set(nltk.corpus.stopwords.words('english'))
    def __iter__(self):
        def comment_to_wordlist(comment, remove_stopwords=True):
            comment = str(comment)
            words = comment.lower().split()
            return words
        for line in open(self.filename, 'r'):
            tokenized_comment = comment_to_wordlist(line, tokenizer)
            yield tokenized_comment
all_comments = FileToComments('all_comments.csv')

from gensim.models import Phrases
from gensim.models.phrases import Phraser
bigram = Phrases(all_comments, min_count=30, threshold=15)
bigram_phraser = Phraser(bigram)

all_tokens = [bigram_phraser[comment] for comment in all_comments]
stops = set(stopwords.words('english'))
clean_all_tokens = []
for token in all_tokens:
    words = [w for w in token if not w in stops]
    clean_all_tokens += [words]

import pickle
with open('tokenized_all_comments.pickle', 'wb') as filename:
    pickle.dump(clean_all_tokens, filename, protocol=pickle.HIGHEST_PROTOCOL)
print('files saved to tokenized_all_comments.pickle...')