text_plain_output_1.png image_output_1.png
from nltk.stem import WordNetLemmatizer, SnowballStemmer

from tensorflow.keras.layers import Embedding, Dense, GlobalAveragePooling1D

from tensorflow.keras.preprocessing.sequence import pad_sequences

from tensorflow.keras.preprocessing.text import Tokenizer

import collections

import matplotlib.pyplot as plt

import pandas as pd

import re

import string

import tensorflow as tf

df_train = pd.read_csv('/kaggle/input/amazon-reviews/train.csv')
df_test = pd.read_csv('/kaggle/input/amazon-reviews/test.csv')

df_train.columns = ['label', 'title', 'text']

df_test.columns = ['label', 'title', 'text']

df_train = df_train.head(100000)
df_test = df_test.head(10000)

def concat_columns(df, col1, col2, new_col):
    df[new_col] = df[col1].apply(str) + ' ' + df[col2].apply(str)
    df.drop(col2, axis=1, inplace=True)
    return df
df_train = concat_columns(df_train, 'text', 'title', 'text')
df_test = concat_columns(df_test, 'text', 'title', 'text')

def clean_text(text):
    text = re.sub('[^A-Za-zÀ-ú ]+', '', text)
    text = re.sub('book|one', '', text)
    text = text.lower()
    text = text.translate(str.maketrans('', '', string.punctuation))
    text = re.sub('\\s+', ' ', text).strip()
    return text
df_train['text'] = df_train['text'].apply(clean_text)
df_test['text'] = df_test['text'].apply(clean_text)

def normalize_text(text):
    stemmer = SnowballStemmer('english')
    normalized_text = []
    for word in text.split():
        stemmed_word = stemmer.stem(word)
        normalized_text.append(stemmed_word)
    return ' '.join(normalized_text)
df_train['text'] = df_train['text'].apply(normalize_text)
df_test['text'] = df_test['text'].apply(normalize_text)

words = []
for text in df_train['text']:
    words.extend(text.split())
word_count = collections.Counter(words)
top_words = dict(word_count.most_common(10))
plt.xticks(range(len(top_words)), list(top_words.keys()))

words = []
for text in df_test['text']:
    words.extend(text.split())
word_count = collections.Counter(words)
top_words = dict(word_count.most_common(10))
plt.xticks(range(len(top_words)), list(top_words.keys()))

max_words = 10000
max_len = 200
tokenizer = Tokenizer(num_words=max_words)
tokenizer.fit_on_texts(df_train['text'])
sequences_train = tokenizer.texts_to_sequences(df_train['text'])
sequences_val = tokenizer.texts_to_sequences(df_test['text'])
word_index = tokenizer.word_index

data_train = pad_sequences(sequences_train, maxlen=max_len)
data_val = pad_sequences(sequences_val, maxlen=max_len)

model = tf.keras.Sequential()
model.add(Embedding(max_words, 16, input_length=max_len))
model.add(GlobalAveragePooling1D())
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
model.summary()

history = model.fit(data_train, df_train['label'], epochs=15, batch_size=64, validation_data=(data_val, df_test['label']))

plt.style.use('dark_background')
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('Model loss')
plt.xlabel('Epoch')
plt.legend(['Training', 'Validation'], loc='upper right')
plt.show()