application_vnd.jupyter.stderr_output_1.png
from html.parser import HTMLParser

from nltk.corpus import stopwords

from nltk.stem import WordNetLemmatizer

from nltk.tokenize import word_tokenize

from wordcloud import WordCloud, STOPWORDS

import collections

import gensim

import nltk

import pandas as pd

import pandas as pd

import re

import scipy.cluster.hierarchy as shc

import unicodedata

import unicodedata
from html.parser import HTMLParser
class HTMLStripper(HTMLParser):
    def __init__(self):
        HTMLParser.__init__(self)
        self._lines = []
    def error(self, message):
        pass
    @staticmethod
    def remove_control_characters(s):
        return ''.join((ch for ch in s if unicodedata.category(ch)[0] != 'C'))
    def read(self, data):
        self.reset()
        self.feed(data)
        return ' '.join(self._lines)
    def handle_data(self, data):
        data = self.remove_control_characters(data)
        data = data.strip()
        self._lines.append(data)

import pandas as pd
class ImportData:
    @staticmethod
    def read_data(file_name):
        data = pd.read_csv(file_name)
        try:
            data = data.drop(['Unnamed: 0'], axis=0)
        except KeyError:
            pass
        return data
    @staticmethod
    def handle_null_values(data):
        data.drop(columns=['product_meta_keywords'], inplace=True)
        data.dropna(axis=0, inplace=True)
        return data
    def import_data(self, csv_file):
        data = self.read_data(csv_file)
        data = self.handle_null_values(data)
        return data

import re
import nltk
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
class Preprocess:
    def __init__(self):
        nltk.download('stopwords')
        self.STOPWORDS = set(stopwords.words('english'))
        self.STRIP_PUNCTUATION = re.compile('[",\\(\\)\\|!\\?;:]')
        self.STRIP_PERIODS = re.compile('\\.(?!\\d)')
    @staticmethod
    def strip_tags(html):
        s = HTMLStripper()
        return s.read(html)
    def normalize(self, text):
        text = text.lower()
        text = self.STRIP_PUNCTUATION.sub('', text)
        text = self.STRIP_PERIODS.sub('', text)
        text = text.split(' ')
        text = [word for word in text if word not in self.STOPWORDS]
        text = ' '.join(text)
        text = [WordNetLemmatizer().lemmatize(word) for word in text]
        text = ''.join(text)
        text = text.strip()
        text = re.sub(' +', ' ', text)
        return text
    def process(self, html):
        text = self.strip_tags(html)
        text = self.normalize(text)
        return text

from wordcloud import WordCloud, STOPWORDS
from matplotlib import pyplot as plt
class Visualize:
    def __init__(self, width=500, height=500, bg_color='white', font_size=15, stopwords=STOPWORDS, text=''):
        self.width = width
        self.height = height
        self.bg_color = bg_color
        self.font_size = font_size
        self.stopwords = stopwords
        self.text = text
        self.wordcloud = WordCloud(width=self.width, height=self.height, background_color=self.bg_color, stopwords=self.stopwords, min_font_size=self.font_size)
    def plot(self):
        wordcloud = self.wordcloud.generate(self.text)
        plt.axis('off')
        plt.tight_layout(pad=0)

import gensim
import pandas as pd
import collections
from nltk.tokenize import word_tokenize
class Doc2Vec:
    def __init__(self, filename='../input/Product_Details.csv'):
        self.data = pd.read_csv(filename)
        self.train_corpus = self.get_train_data()
        self.test_corpus = self.get_test_data()
    def read_corpus(self, tokens_only=False):
        for i, row in enumerate(self.data):
            if tokens_only:
                yield word_tokenize(row)
            else:
                yield gensim.models.doc2vec.TaggedDocument(word_tokenize(row), [i])
    def get_train_data(self):
        train_corpus = list(self.read_corpus())
        return train_corpus
    def get_test_data(self):
        test_corpus = list(self.read_corpus(tokens_only=True))
        return test_corpus
    def train_model(self, epochs=40, vector_size=50, min_count=2, workers=4, save_model=True):
        model = gensim.models.doc2vec.Doc2Vec(vector_size=vector_size, min_count=min_count, epochs=epochs, workers=workers)
        model.build_vocab(self.train_corpus)
        model.train(self.train_corpus, total_examples=model.corpus_count, epochs=model.epochs)
        if save_model:
            name = 'doc2vec.model'
            model.save(name)
        return model
    def test_model(self):
        model = self.train_model()
        ranks = []
        second_ranks = []
        for doc_id in range(len(self.train_corpus)):
            inferred_vector = model.infer_vector(self.train_corpus[doc_id].words)
            sims = model.docvecs.most_similar([inferred_vector], topn=len(model.docvecs))
            rank = [docid for docid, sim in sims].index(doc_id)
            ranks.append(rank)
            second_ranks.append(sims[1])
        collections.Counter(ranks)
    def export_vsm_vocab(self, model):
        document_vector_list = []
        for i in range(self.data.shape[0]):
            document_vector_list.append(model.docvecs[i])
        return document_vector_list

data_importer = ImportData()
data = data_importer.import_data('../input/qiagen-detail/Qiagen_details.csv')

def TaggedDoc(dataframe):
    global punct
    tagdoc = []
    for i in range(len(dataframe)):
        x = dataframe.iloc[i]
        y = gensim.models.doc2vec.TaggedDocument(x.lower().split(), [i])
        tagdoc.append(y)
    return tagdoc

df_desc = TaggedDoc(data['product_html'])

model = gensim.models.doc2vec.Doc2Vec(vector_size=50, max_count=5, epochs=30, dm=1, workers=4, dbow_words=0)

model.build_vocab(df_desc)

len(model.wv.vocab)

import scipy.cluster.hierarchy as shc
plt.figure(figsize=(10, 8))
plt.title('Dendrograms')
dend = shc.dendrogram(shc.linkage(model.docvecs.vectors_docs, method='ward'))