image_output_1.png
from nltk import FreqDist

import matplotlib.pyplot as plt

import pandas as pd

import seaborn as sns

toxic_comment_processed_seqlen = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train-processed-seqlen128.csv')
toxic_comment = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv')
unintended_bias_preprocessed = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train-processed-seqlen128.csv')
unintended_bias = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv')

toxic_percentage = toxic_comment['toxic'].value_counts(normalize=True) * 100
unintended_bias_toxic_percentage = (unintended_bias['toxic'] > 0.5).value_counts(normalize=True) * 100

plt.xticks([0, 1], ['Non-toxic', 'Toxic'], rotation=0)
plt.xticks([0, 1], ['Non-toxic', 'Toxic'], rotation=0)
plt.tight_layout()

all_words = []
for words in toxic_comment['tokenized_words']:
    all_words.extend(words)

freq_dist = FreqDist(all_words)

all_words = []
for words in toxic_comment['tokenized_words']:
    all_words.extend(words)

freq_dist = FreqDist(all_words)
plt.figure(figsize=(12, 6))
freq_dist.plot(20, cumulative=False)
plt.show()