text_plain_output_1.png
from gensim.models import KeyedVectors

from sklearn.decomposition import TruncatedSVD

from sklearn.feature_extraction.text import TfidfVectorizer

import numpy as np

import numpy as np

import pandas as pd

import spacy

folder_path_train = '/content/drive/MyDrive/HMD_project/train.jsonl'
folder_path_dev = '/content/drive/MyDrive/HMD_project/dev.jsonl'
df_train = pd.read_json(folder_path_train, lines=True)
df_dev = pd.read_json(folder_path_dev, lines=True)

with open('/content/drive/MyDrive/HMD_project/new/text_aug_norm.txt', 'r') as f:
    lines = f.readlines()
    file_names = []
    for i in lines:
        file_names.append(i[:i.find('\n')])

for i in range(len(file_names)):
    df_train.loc[len(df_train.index)] = [1, 'img/1', 1, file_names[i]]

data = df_train

from gensim.models import KeyedVectors
wv = KeyedVectors.load('/content/drive/MyDrive/HMD_project/glove-twitter-200')

import spacy.cli
spacy.cli.download('en_core_web_sm')
nlp = spacy.load('en_core_web_sm')

def preprocess(text):
    doc = nlp(text)
    filtered_token = []
    for token in doc:
        if token.is_punct or token.is_space or token.is_bracket or token.is_stop:
            continue
        else:
            token = token.lemma_
            filtered_token.append(token)
    return filtered_token

df_dev['processed_text_val'] = df_dev['text'].apply(lambda x: preprocess(x))
data['processed_text'] = data['text'].apply(lambda x: preprocess(x))

import numpy as np
def gensim_vector(token):
    vec_size = wv.vector_size
    wv_final = np.zeros(vec_size)
    count = 1
    for t in token:
        if t in wv:
            count += 1
            wv_final += wv[t]
    return wv_final / count

data['text_vector'] = data['processed_text'].apply(gensim_vector)
df_dev['text_vector_val'] = df_dev['processed_text_val'].apply(gensim_vector)

len(data.text_vector.iloc[0])

vectorizer = TfidfVectorizer(stop_words='english')
svd = TruncatedSVD(n_components=1000)
processed_text = data['processed_text'].apply(lambda x: ' '.join(x))
text_vec_tfidf = vectorizer.fit_transform(processed_text)
print('CountVectorizer shape_training dataset', text_vec_tfidf.shape)
processed_text_val = df_dev['processed_text_val'].apply(lambda x: ' '.join(x))
text_vec_tfidf_val = vectorizer.transform(processed_text_val)
print('CountVectorizer shape_val dataset', text_vec_tfidf_val.shape)
lsa_text = svd.fit_transform(text_vec_tfidf)
print('\nvariance_captured_by 1000 components', svd.explained_variance_ratio_.sum())
lsa_text_val = svd.transform(text_vec_tfidf_val)
print('\n', lsa_text.shape)
print(lsa_text_val.shape)