text_plain_output_1.png
from gensim.models import Word2Vec

from keras.preprocessing.text import Tokenizer

import numpy as np

import pandas as pd

datasets_dir = ''
vnrows = None
datasets_dir = '../input/'
df = pd.read_csv(datasets_dir + 'fakenews_preprocessed_35k.csv', nrows=vnrows, encoding='utf-8')

df = df.loc[:, ~df.columns.str.contains('^Unnamed')]

df = df.sample(frac=1).reset_index(drop=True)

max_seq_len = 458

MAX_NB_WORDS = round(int(df.doc_length.sum() / max_seq_len))

tokenizer = Tokenizer(num_words=MAX_NB_WORDS, lower=True, char_level=False)

tokenizer.fit_on_texts(X_train)
word_index = tokenizer.word_index

gensim_news_desc = []
chunk_data = X_train
for record in range(0, len(chunk_data)):
    news_desc_list = []
    for tok in chunk_data[record].split():
        news_desc_list.append(str(tok))
    gensim_news_desc.append(news_desc_list)

vsize = max_seq_len
vmin_count = 4
gensim_model = Word2Vec(gensim_news_desc, min_count=vmin_count, size=vsize, sg=1)

words = list(gensim_model.wv.vocab)

batch_size = 1024
num_epochs = 10
num_filters = 128
embed_dim = max_seq_len
weight_decay = 0.0001
class_weight = {0: 1, 1: 1}
print('preparing embedding matrix...')
gensim_words_not_found = []
gensim_nb_words = len(gensim_model.wv.vocab)
print('gensim_nb_words : ', gensim_nb_words)
gensim_embedding_matrix = np.zeros((gensim_nb_words, embed_dim))
for word, i in word_index.items():
    if i >= gensim_nb_words:
        continue
    if word in gensim_model.wv.vocab:
        embedding_vector = gensim_model.wv[word]
        if embedding_vector is not None and len(embedding_vector) > 0:
            gensim_embedding_matrix[i] = embedding_vector
    else:
        gensim_words_not_found.append(word)
print(gensim_embedding_matrix.shape)