text_plain_output_1.png
from gensim.corpora import Dictionary

from gensim.models import TfidfModel, LsiModel, Word2Vec

from nltk import sent_tokenize, word_tokenize

from nltk.corpus import stopwords

import re

stop_words = stopwords.words('english')
def clean_text(txt: str):
    """Clean and lower case text."""
    txt = re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower())
    txt = re.sub('\\b\\d+\\b', '', txt).strip()
    return txt
def tokenizer(txt: str):
    """Custom tokenizer."""
    tokens = []
    for sent in sent_tokenize(txt, language='english'):
        for word in word_tokenize(clean_text(sent), language='english'):
            if len(word) < 2:
                continue
            if word in stop_words:
                continue
            tokens.append(word)
    return tokens

tokenized_data = [tokenizer(doc) for doc in data]
dct = Dictionary(tokenized_data)

rare_ids = [tokenid for tokenid, wordfreq in dct.cfs.items() if wordfreq < 2]
dct.filter_tokens(rare_ids)

corpus = [dct.doc2bow(line) for line in tokenized_data]
tfidf_model = TfidfModel(corpus, id2word=dct)
tfidf_matrix = tfidf_model[corpus]

print('Size of LSI vocab.:', len(dct.keys()))
print('Size of w2v vocab.:', len(w2v_model.wv.key_to_index.keys()))