text_plain_output_1.png
from sklearn.decomposition import TruncatedSVD

from sklearn.metrics.pairwise import cosine_similarity

import pandas as pd

datasets_to_combine = [('/kaggle/input/uci-bag-of-words/docword.enron.txt', '/kaggle/input/uci-bag-of-words/vocab.enron.txt'), ('/kaggle/input/uci-bag-of-words/docword.kos.txt', '/kaggle/input/uci-bag-of-words/vocab.kos.txt'), ('/kaggle/input/uci-bag-of-words/docword.nips.txt', '/kaggle/input/uci-bag-of-words/vocab.nips.txt')]
dataset_column_names = ['documentID', 'wordID', 'count']
merged_docwords = []
for dataset_tuple in datasets_to_combine:
    docword = pd.read_csv(dataset_tuple[0], delim_whitespace=True, header=None, skiprows=3, names=dataset_column_names, nrows=100000)
    vocab = pd.read_csv(dataset_tuple[1], delim_whitespace=True, header=None, names=['word'])
    merged = pd.merge(docword, vocab.reset_index(), how='inner', left_on='wordID', right_on='index').drop('index', axis=1)
    merged_docwords.append(merged)

corpus = pd.concat(merged_docwords, axis=0, ignore_index=True)

wdm = pd.pivot_table(corpus, values='count', index='word', columns='documentID', fill_value=0)

svd = TruncatedSVD(n_components=100, random_state=7)
svd.fit(wdm)

wdm_transformed = pd.DataFrame(svd.transform())

wdm_enron = pd.pivot_table(merged_docwords[0], values='count', index='word', columns='documentID', fill_value=0)
svd = TruncatedSVD(n_components=100, random_state=7)
wdm_enron_transformed = svd.fit_transform(wdm_enron)

wdm_kos = pd.pivot_table(merged_docwords[1], values='count', index='word', columns='documentID', fill_value=0)
svd = TruncatedSVD(n_components=100, random_state=7)
wdm_kos_transformed = svd.fit_transform(wdm_kos)

kos_similarity = cosine_similarity(wdm_kos_transformed)
kos_similarity.mean()