text_plain_output_1.png
from sklearn.decomposition import PCA

import csv

import matplotlib.pyplot as plt

import numpy as np

import seaborn as sns

import skfuzzy as fuzz

ROOT_PATH = '/kaggle/input/CORD-19-research-challenge/'
METADATA_PATH = f'{ROOT_PATH}/metadata.csv'
SAMPLE_SIZE = None

def create_embedding_dict(filepath, sample_size=None):
    """function to create embedding dictionary using given file"""
    embedding_dict = {}
    with open(filepath) as csvfile:
        reader = csv.reader(csvfile, delimiter=',')
        for i, row in enumerate(reader):
            if i == sample_size:
                break
            embed = np.zeros((768,))
            for idx, val in enumerate(row):
                if idx > 0:
                    embed[idx - 1] = float(val)
            embedding_dict[row[0]] = embed
    return embedding_dict

embedding_dict = create_embedding_dict(f'{ROOT_PATH}/cord19_specter_embeddings_2020-04-10/cord19_specter_embeddings_2020-04-10.csv', sample_size=SAMPLE_SIZE)
embedding_mat = np.array(list(embedding_dict.values()))
(embedding_mat.shape, len(embedding_dict))

n_clusters = 10

def fuzzy_clustering(all_embedding, n_clusters):
    centroids, u, u0, d, jm, p, fpc = fuzz.cluster.cmeans(data=all_embedding.T, c=n_clusters, m=2, error=0.5, maxiter=100, init=None)
    clusters = np.argmax(u, axis=0)
    return (clusters, centroids)

def get_pca(all_embedding):
    pca = PCA()
    pca_result = pca.fit_transform(all_embedding)
    return pca_result

def plot_pca(pca_result, clusters, title):
    sns.set(rc={'figure.figsize': (10, 10)})
    palette = sns.color_palette('bright', len(set(clusters)))

fuzzy_pca = get_pca(embedding_mat)

plot_pca(fuzzy_pca, fuzzy_clusters, 'PCA Covid-19 Articles - Clustered(Fuzzy C-Means)')