text_html_output_1.png
from nltk.corpus import stopwords

from sklearn.cluster import MiniBatchKMeans

from sklearn.feature_extraction.text import TfidfVectorizer

from sklearn.preprocessing import StandardScaler, OrdinalEncoder

from tqdm import tqdm

import matplotlib.pyplot as plt

import pandas as pd

df_reviews_raw = pd.read_csv('/kaggle/input/dataset-of-malicious-and-benign-webpages/Webpages_Classification_train_data.csv/Webpages_Classification_train_data.csv').drop(['Unnamed: 0'], axis=1)

df_reviews_raw.isna().sum()

df_reviews_raw.dtypes

df_reviews_untrimmed_sample = df_reviews_raw.groupby('label').apply(lambda x: x.sample(25000, random_state=42)).reset_index(drop=True)
df_reviews_trimmed = df_reviews_untrimmed_sample[df_reviews_untrimmed_sample.content.str.split().str.len().ge(60)]

df_reviews_sampled = df_reviews_trimmed.groupby('label').apply(lambda x: x.sample(2000, random_state=42)).reset_index(drop=True)
df_reviews = df_reviews_sampled.sample(frac=1, random_state=42).reset_index(drop=True)

df_reviews['geo_loc'] = OrdinalEncoder().fit_transform(df_reviews.geo_loc.values.reshape(-1, 1))
df_reviews['tld'] = OrdinalEncoder().fit_transform(df_reviews.tld.values.reshape(-1, 1))
df_reviews['who_is'] = OrdinalEncoder().fit_transform(df_reviews.who_is.values.reshape(-1, 1))
df_reviews['https'] = OrdinalEncoder().fit_transform(df_reviews.https.values.reshape(-1, 1))
df_reviews['label'] = OrdinalEncoder().fit_transform(df_reviews.label.values.reshape(-1, 1))
df_reviews['url'] = df_reviews.url.apply(lambda x: ' '.join(x.split('://')[1].strip('www.').replace('.', '/').split('/')))

tqdm.pandas()
stop = stopwords.words()
df_reviews.content = df_reviews.content.str.replace('[^\\w\\s]', '').str.lower()
df_reviews.content = df_reviews.content.progress_apply(lambda x: ' '.join([item for item in x.split() if item not in stop]))
df_reviews.url = df_reviews.url.str.replace('[^\\w\\s]', '').str.lower()
df_reviews.url = df_reviews.url.progress_apply(lambda x: ' '.join([item for item in x.split() if item not in stop]))

tfidf = TfidfVectorizer(min_df=5, max_df=0.95, max_features=8000, stop_words='english')
tfidf.fit(df_reviews.url)
url_tfidf = tfidf.transform(df_reviews.url)
tfidf.fit(df_reviews.content)
content_tfidf = tfidf.transform(df_reviews.content)

def find_optimal_clusters(data, max_k):
    k_list = range(2, max_k+1)
    
    sse = []
    for k in k_list:
        sse.append(MiniBatchKMeans(n_clusters=k, init_size=1024, batch_size=2048, random_state=20).fit(data).inertia_)
       
    plt.style.use("dark_background")
    f, ax = plt.subplots(1, 1)
    ax.plot(k_list, sse, marker='o')
    ax.set_xlabel('Cluster Centers')
    ax.set_xticks(k_list)
    ax.set_xticklabels(k_list)
    ax.set_ylabel('SSE')
    ax.set_title('SSE by Cluster Center Plot')

find_optimal_clusters(url_tfidf, 20)