text_html_output_1.png application_vnd.jupyter.stderr_output_1.png
from collections import OrderedDict

from gensim import corpora

from gensim import corpora, models, similarities

from nltk.corpus import stopwords

from string import punctuation

from subprocess import check_output

import os

import pandas as pd

import tempfile

import os
import pandas as pd
from gensim import corpora, models, similarities
from subprocess import check_output
import gensim
import logging
import tempfile
TEMP_FOLDER = tempfile.gettempdir()
from gensim import corpora
from nltk.corpus import stopwords
from string import punctuation
df = pd.DataFrame()
df = pd.read_csv('../input/winemag-data-130k-v2.csv')
df.shape

corpus = []
a = []
for i in range(len(df['description'])):
    a = df['description'][i]
    corpus.append(a)
corpus[0:2]

stoplist = stopwords.words('english') + list(punctuation)
texts = [[word for word in str(document).lower().split() if word not in stoplist] for document in corpus]
dictionary = corpora.Dictionary(texts)
dictionary.save(os.path.join(TEMP_FOLDER, 'wine.dict'))
corpus = [dictionary.doc2bow(text) for text in texts]
corpora.MmCorpus.serialize(os.path.join(TEMP_FOLDER, 'wine.mm'), corpus)
tfidf = models.TfidfModel(corpus)
corpus_tfidf = tfidf[corpus]
total_topics = 20
lda = models.LdaModel(corpus, id2word=dictionary, num_topics=total_topics)
corpus_lda = lda[corpus_tfidf]

from collections import OrderedDict
data_lda = {i: OrderedDict() for i in range(total_topics)}
df_lda = pd.DataFrame(data_lda)
df_lda = df_lda.fillna(0).T

import pyLDAvis.gensim
pyLDAvis.enable_notebook()
panel = pyLDAvis.gensim.prepare(lda, corpus_lda, dictionary, mds='tsne')
panel