text_html_output_1.png
from itertools import chain

from scipy.cluster.hierarchy import fcluster

from sklearn.feature_extraction.text import TfidfVectorizer

import matplotlib.patches as patches

import matplotlib.pyplot as plt

import numpy as np

import pandas as pd

import seaborn as sns

df = pd.read_csv('/kaggle/input/cleaning-cord-19-metadata/cord_metadata_cleaned.csv')
df = df.dropna(subset=['text_simplified']).reset_index(drop=True)

tokens = df['text_simplified'].str.split(' ').tolist()
tokens = pd.Series(chain(*tokens))
tokens_count = tokens.value_counts()
tokens_count

ax = tokens_count.plot(figsize=(15, 5))
ax.set_xlabel("Token")
ax.set_ylabel("Count")
ax.grid(True)

ax = tokens_count[10000:].plot(figsize=(15, 5))
ax.set_xlabel("Token")
ax.set_ylabel("Count"),
ax.grid(True)

tfidf_vectorizer = TfidfVectorizer(input='content', lowercase=False, preprocessor=lambda text: text, tokenizer=lambda text: text.split(' '), token_pattern=None, analyzer='word', stop_words=None, ngram_range=(1, 1), max_features=10000, binary=False, norm='l2', use_idf=True, smooth_idf=True, sublinear_tf=False)
features = tfidf_vectorizer.fit_transform(df['text_simplified'])
features.shape

features = features.astype('float32').toarray()

sample_size = 0.1
sample_mask = np.random.choice(a=[True, False], size=len(features), p=[sample_size, 1 - sample_size])
features_sample = features[sample_mask]
features_sample.shape

sns.clustermap(data=distance_matrix, col_linkage=Z, row_linkage=Z, cmap=plt.get_cmap('RdBu'))

dissimilarities = pd.Series(distance_matrix.flatten())

ax = dissimilarities.hist(bins=100, figsize=(15, 5))
ax.set_xlabel("Cosine dissimilarity")
ax.set_ylabel("Count")
ax.grid(True)

ax = dissimilarities[dissimilarities >= 0.8].hist(bins=100, figsize=(15, 5))
ax.set_xlabel("Cosine dissimilarity")
ax.set_ylabel("Count")
ax.grid(True)

ax = dissimilarities[dissimilarities >= 0.95].hist(bins=100, figsize=(15, 5))
ax.set_xlabel("Cosine dissimilarity")
ax.set_ylabel("Count")
ax.grid(True)

clusters = fcluster(Z, t=0.999, criterion='distance')
clustermap = sns.clustermap(data=distance_matrix, col_linkage=Z, row_linkage=Z, cmap=plt.get_cmap('RdBu'))
cluster_mapping = dict(zip(range(len(features_sample)), clusters))
clustermap_clusters = pd.Series([cluster_mapping[id_] for id_ in list(clustermap.data2d.columns)])
for cluster in set(clusters):
    cluster_range = list(clustermap_clusters[clustermap_clusters == cluster].index)
    clustermap.ax_heatmap.add_patch(patches.Rectangle(xy=(np.min(cluster_range), np.min(cluster_range)), width=len(cluster_range), height=len(cluster_range), fill=False, edgecolor='lightgreen', lw=2))

cluster_count = df['cluster'].value_counts().sort_values()

ax = cluster_count.plot(kind='bar', figsize=(15, 5))
ax.set_xticks([])
ax.set_xlabel("Cluster id")
ax.set_ylabel("Count")
ax.grid(True)

noise_clusters = set(cluster_count[cluster_count <= 5].index)
noise_mask = df['cluster'].isin(noise_clusters)
df.loc[noise_mask, 'cluster'] = -1

columns = np.array(tfidf_vectorizer.get_feature_names())
top_k = 3
def describe(df: pd.DataFrame) -> pd.DataFrame:
    order = features[df.index].mean(axis=0).argsort()[::-1][:top_k]
    top_words = columns[order]
    cluster_id = df['cluster'].iloc[0]
    for i, word in enumerate(top_words):
        df[f'word_{i + 1}'] = word if cluster_id != -1 else ''
    return df
df = df.groupby('cluster').apply(describe)

df.filter(regex='text_simplified|word_\\d+', axis=1)