text_html_output_1.png
from nltk.stem import PorterStemmer

import gensim

import nltk

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

path = '../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'
embeddings = gensim.models.KeyedVectors.load_word2vec_format(path, binary=True)

pd.Series(embeddings['modi'][:5])

embeddings.most_similar('modi', topn=10)

url = 'https://bit.ly/2S2yXEd'
data = pd.read_csv(url)

doc1 = data.iloc[0, 0]

docs = data['review']

words = nltk.word_tokenize(doc1.lower())
temp = pd.DataFrame()
for word in words:
    try:
        temp = temp.append(pd.Series(embeddings[word][:5]), ignore_index=True)
    except:

docs = docs.str.lower().str.replace('[^a-z ]', '')

from nltk.stem import PorterStemmer
stemmer = PorterStemmer()
stopwords = nltk.corpus.stopwords.words('english')
def clean_doc(doc):
    words = doc.split(' ')
    words_clean = [word for word in words if word not in stopwords]
    doc_clean = ' '.join(words_clean)
    return doc_clean
docs_clean = docs.apply(clean_doc)

docs_clean.shape

docs_vectors = pd.DataFrame()
for doc in docs_clean:
    words = nltk.word_tokenize(doc)
    temp = pd.DataFrame()
    for word in words:
        try:
            word_vec = embeddings[word]
            temp = temp.append(pd.Series(word_vec), ignore_index=True)
        except:
            pass
    docs_vectors = docs_vectors.append(temp.mean(), ignore_index=True)
docs_vectors.shape