text_html_output_1.png
from nltk.corpus import stopwords

from sklearn.model_selection import train_test_split

from sklearn.model_selection import train_test_split

import numpy as np

import numpy as np # linear algebra

import pandas as pd

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import re

import string

import tensorflow as tf

import torch

import torch

import torch.nn as nn

df = pd.read_csv('/kaggle/input/sentiment140/training.1600000.processed.noemoticon.csv', encoding='latin', header=None)
df.columns = ['sentiment', 'id', 'date', 'query', 'user_id', 'text']
df = df.drop(['id', 'date', 'query', 'user_id'], axis=1)

df.sentiment = df.sentiment.map({0: 0, 4: 1})

stop_words = set(stopwords.words('english'))
def data_preprocessing(text):
    text = text.lower()
    text = re.sub('<.*?>', '', text)
    text = ''.join([c for c in text if c not in string.punctuation])
    text = [word for word in text.split() if word not in stop_words]
    text = ' '.join(text)
    return text
df['text'] = df['text'].astype(str).apply(data_preprocessing)

MAX_LEN = max([len(x.split()) for x in df['text']])

import torch
class Dataset:
    def __init__(self, text, sentiment):
        self.text = text
        self.sentiment = sentiment
    def __len__(self):
        return len(self.text)
    def __getitem__(self, item):
        text = self.text[item, :]
        target = self.sentiment[item]
        return {'text': torch.tensor(text, dtype=torch.long), 'target': torch.tensor(target, dtype=torch.long)}

def load_vectors(fname):
    fin = open(fname)
    data = {}
    for line in fin:
        tokens = line.split()
        data[tokens[0]] = np.array([float(value) for value in tokens[1:]])
    return data

def create_embedding_matrix(word_index, embedding_dict):
    """
    This function creates the embedding matrix
    :param word_index: a dictionary of word: index_value
    :param embedding_dict:
    :return a numpy array with embedding vectors for all known words
    """
    embedding_matrix = np.zeros((len(word_index) + 1, 300))
    for word, i in word_index.items():
        if word in embedding_dict:
            embedding_matrix[i] = embedding_dict[word]
    return embedding_matrix

class sentimentBiLSTM(nn.Module):
    """
    The RNN model that will be used to perform Sentiment analysis.
    """
    def __init__(self, embedding_matrix, hidden_dim, output_size):
        """
        Initialize the model by setting up the layers.
        """
        super(sentimentBiLSTM, self).__init__()
        self.embedding_matrix = embedding_matrix
        self.hidden_dim = hidden_dim
        num_words = self.embedding_matrix.shape[0]
        embed_dim = self.embedding_matrix.shape[1]
        self.embedding = nn.Embedding(num_embeddings=num_words, embedding_dim=embed_dim)
        self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
        self.embedding.weight.requires_grad = False
        self.lstm = nn.LSTM(embed_dim, hidden_dim, bidirectional=True, batch_first=True)
        self.fc = nn.Linear(hidden_dim * 2, output_size)
        self.sigmoid = nn.Sigmoid()
    def forward(self, x):
        batch_size = x.size(0)
        embeds = self.embedding(x)
        lstm_out, _ = self.lstm(embeds)
        lstm_out = lstm_out.contiguous().view(-1, self.hidden_dim * 2)
        out = self.fc(lstm_out)
        sig_out = self.sigmoid(out)
        sig_out = sig_out.view(batch_size, -1)
        sig_out = sig_out[:, -1]
        return sig_out

from sklearn.model_selection import train_test_split
y = df.sentiment.values
train_df, test_df = train_test_split(df, test_size=0.2, stratify=y)
import tensorflow as tf
BATCH_SIZE = 16
print('Fitting tokenizer')
tokenizer = tf.keras.preprocessing.text.Tokenizer()
tokenizer.fit_on_texts(df.text.values.tolist())
xtrain = tokenizer.texts_to_sequences(train_df.text.values)
xtest = tokenizer.texts_to_sequences(test_df.text.values)
xtrain = tf.keras.preprocessing.sequence.pad_sequences(xtrain, maxlen=MAX_LEN)
xtest = tf.keras.preprocessing.sequence.pad_sequences(xtest, maxlen=MAX_LEN)
train_dataset = Dataset(text=xtrain, sentiment=train_df.sentiment.values)
train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, drop_last=True)
valid_dataset = Dataset(text=xtest, sentiment=test_df.sentiment.values)
valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, drop_last=True)