text_html_output_1.png
from sklearn.cluster import KMeans

from sklearn.decomposition import LatentDirichletAllocation

from sklearn.decomposition import PCA

from sklearn.feature_extraction.text import CountVectorizer

from sklearn.feature_extraction.text import TfidfVectorizer

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

root_path = '/kaggle/input/medieval-chant-corpus'
df_path = f'{root_path}/mh-corpus.json'
df = pd.read_json(df_path)
df.drop(['rawTextMusic', 'rawText'], axis=1, inplace=True)

from sklearn.feature_extraction.text import TfidfVectorizer
maxx_features = 2 ** 12
vectorizer = TfidfVectorizer(max_features=maxx_features)
X = vectorizer.fit_transform(df['music_as_words'].values)

from sklearn.decomposition import PCA
pca = PCA(n_components=0.95, random_state=42)
X_reduced = pca.fit_transform(X.toarray())
X_reduced.shape

k = 15
kmeans = KMeans(n_clusters=k, random_state=42)
y_pred = kmeans.fit_predict(X_reduced)
df['y'] = y_pred

vectorizers = []
for ii in range(0, k):
    vectorizers.append(CountVectorizer(min_df=5, max_df=0.9, stop_words='english', lowercase=True, token_pattern='[a-zA-Z\\-][a-zA-Z\\-]{2,}'))

vectorized_data = []
for current_cluster, cvec in enumerate(vectorizers):
    try:
        vectorized_data.append(cvec.fit_transform(df.loc[df['y'] == current_cluster, 'music_as_words']))
    except Exception as e:
        vectorized_data.append(None)

NUM_TOPICS_PER_CLUSTER = 20
lda_models = []
for ii in range(0, k):
    lda = LatentDirichletAllocation(n_components=NUM_TOPICS_PER_CLUSTER, max_iter=10, learning_method='online', verbose=False, random_state=42)
    lda_models.append(lda)
lda_models[0]

clusters_lda_data = []
for current_cluster, lda in enumerate(lda_models):
    print('Current Cluster: ' + str(current_cluster))
    if vectorized_data[current_cluster] != None:
        clusters_lda_data.append(lda.fit_transform(vectorized_data[current_cluster]))