text_html_output_1.png
from nltk.stem import WordNetLemmatizer, SnowballStemmer

import collections

import matplotlib.pyplot as plt

import pandas as pd

import re

import string

df_train = pd.read_csv('/kaggle/input/amazon-reviews/train.csv')
df_test = pd.read_csv('/kaggle/input/amazon-reviews/test.csv')

df_train.columns = ['label', 'title', 'text']

df_test.columns = ['label', 'title', 'text']

df_train = df_train.head(100000)
df_test = df_test.head(10000)

def concat_columns(df, col1, col2, new_col):
    df[new_col] = df[col1].apply(str) + ' ' + df[col2].apply(str)
    df.drop(col2, axis=1, inplace=True)
    return df
df_train = concat_columns(df_train, 'text', 'title', 'text')
df_test = concat_columns(df_test, 'text', 'title', 'text')

def clean_text(text):
    text = re.sub('[^A-Za-zÀ-ú ]+', '', text)
    text = re.sub('book|one', '', text)
    text = text.lower()
    text = text.translate(str.maketrans('', '', string.punctuation))
    text = re.sub('\\s+', ' ', text).strip()
    return text
df_train['text'] = df_train['text'].apply(clean_text)
df_test['text'] = df_test['text'].apply(clean_text)

def normalize_text(text):
    stemmer = SnowballStemmer('english')
    normalized_text = []
    for word in text.split():
        stemmed_word = stemmer.stem(word)
        normalized_text.append(stemmed_word)
    return ' '.join(normalized_text)
df_train['text'] = df_train['text'].apply(normalize_text)
df_test['text'] = df_test['text'].apply(normalize_text)

words = []
for text in df_train['text']:
    words.extend(text.split())
word_count = collections.Counter(words)
top_words = dict(word_count.most_common(10))
plt.figure(figsize=(10, 6))
plt.bar(range(len(top_words)), list(top_words.values()), align='center')
plt.xticks(range(len(top_words)), list(top_words.keys()))
plt.grid(alpha=0.5)
plt.title('Top 10 most used words', fontsize=18)
plt.xlabel('Words')
plt.ylabel('Frequency')