image_output_1.png
from itertools import chain

import pandas as pd

df = pd.read_csv('/kaggle/input/cleaning-cord-19-metadata/cord_metadata_cleaned.csv')
df = df.dropna(subset=['text_simplified']).reset_index(drop=True)

tokens = df['text_simplified'].str.split(' ').tolist()
tokens = pd.Series(chain(*tokens))
tokens_count = tokens.value_counts()
tokens_count

ax = tokens_count.plot(figsize=(15, 5))
ax.set_xlabel("Token")
ax.set_ylabel("Count")
ax.grid(True)

ax = tokens_count[10000:].plot(figsize=(15, 5))
ax.set_xlabel("Token")
ax.set_ylabel("Count"),
ax.grid(True)

dissimilarities = pd.Series(distance_matrix.flatten())

ax = dissimilarities.hist(bins=100, figsize=(15, 5))
ax.set_xlabel('Cosine dissimilarity')
ax.set_ylabel('Count')
ax.grid(True)