text_plain_output_1.png
from gensim.models import word2vec

from nltk import word_tokenize

from nltk.corpus import stopwords

from nltk.stem import WordNetLemmatizer

from nltk.stem.snowball import SnowballStemmer

from nltk.tokenize import word_tokenize

from sklearn.manifold import TSNE

from tensorflow.keras.preprocessing.sequence import pad_sequences

from tensorflow.keras.preprocessing.text import Tokenizer

import nltk

import numpy as np

import pandas as pd  # For data handling

import re
import pandas as pd
from time import time
from collections import defaultdict
from bs4 import BeautifulSoup
from nltk import word_tokenize
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from sklearn.decomposition import PCA
from matplotlib import pyplot as plt
from gensim.models import Word2Vec, Phrases
from nltk.tokenize import word_tokenize
import nltk
import seaborn as sns
import numpy as np
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.preprocessing.text import Tokenizer
import gensim
nltk.download('punkt')
nltk.download('wordnet')
nltk.download('omw-1.4')
import spacy

df = pd.read_csv('../input/google-war-news/war-news.csv', encoding='latin1')

df.isnull().sum()

df.dropna(subset=['Summary'], inplace=True)

from nltk.stem.snowball import SnowballStemmer
def stemm_text(text):
    stemmer = SnowballStemmer('english')
    return ' '.join([stemmer.stem(w) for w in text.split(' ')])

w_tokenizer = nltk.tokenize.WhitespaceTokenizer()
lemmatizer = nltk.stem.WordNetLemmatizer()
def lemmatize_text(text):
    lemmatizer = WordNetLemmatizer()
    return ' '.join([lemmatizer.lemmatize(w) for w in text.split(' ')])

T = df['Summary'].str.split(' \n\n---\n\n').str[0]
T = T.str.replace('-', ' ').str.replace('[^\\w\\s]', '').str.replace('\n', ' ').str.lower()
stop = stopwords.words('english')
T = T.apply(lambda x: ' '.join((x for x in x.split() if not x.isdigit())))
T = T.apply(lambda words: ' '.join((word.lower() for word in words.split() if word not in stop)))

T1 = T.values.tolist()

def Sentence2Vec(T, embedding_dim=100, max_length=300):
    glove_path = '../input/glove6b100dtxt/glove.6B.100d.txt'
    path = glove_path
    tokenizer = Tokenizer()
    text = T
    tokenizer.fit_on_texts(text)
    word_index = tokenizer.word_index
    vocab_size = 5000
    trunc_type = 'post'
    oov_tok = '<OOV>'
    padding_type = 'post'
    text_sequence = tokenizer.texts_to_sequences(text)
    text_sequence = pad_sequences(text_sequence, maxlen=max_length, truncating=trunc_type, padding=padding_type)
    size_of_vocabulary = len(tokenizer.word_index) + 1
    embeddings_index = dict()
    f = open(path)
    for line in f:
        values = line.split()
        word = values[0]
        coefs = np.asarray(values[1:], dtype='float32')
        embeddings_index[word] = coefs
    f.close()
    embedding_matrix = np.zeros((size_of_vocabulary, embedding_dim))
    for word, i in tokenizer.word_index.items():
        embedding_vector = embeddings_index.get(word)
        if embedding_vector is not None:
            embedding_matrix[i] = embedding_vector
    text_shape = text_sequence.shape
    X_train = np.empty((text_shape[0], text_shape[1], embedding_matrix.shape[1]))
    for i in range(text_sequence.shape[0]):
        for j in range(text_sequence.shape[1]):
            X_train[i, j, :] = embedding_matrix[text_sequence[i][j]]
    return (X_train, embeddings_index, word_index)

L = list(word_index.keys())

emb_matrix = np.zeros((len(L) + 1, 100))
for word, index in word_index.items():
    embedding_vector = embeddings_index.get(word)
    if embedding_vector is not None:
        emb_matrix[index, :] = embedding_vector

emb_matrix.shape

tsne = TSNE(perplexity=40, n_components=2, init='pca', n_iter=2500, random_state=123)
z = tsne.fit_transform(emb_matrix)

T1 = T.apply(word_tokenize)

num_features = 100
min_word_count = 20
num_workers = 4
context = 20
downsampling = 0.001
from gensim.models import word2vec
model = word2vec.Word2Vec(workers=num_workers, vector_size=num_features, min_count=min_word_count, window=context, sample=downsampling)
model.build_vocab(T1, progress_per=1000)
model.train(T1, total_examples=model.corpus_count, epochs=model.epochs)

labels = model.wv.index_to_key

model.wv['ukraine']

X = model.wv[model.wv.index_to_key]

X.shape

tsne = TSNE(perplexity=40, n_components=2, init='pca', n_iter=2500, random_state=123)
z = tsne.fit_transform(X)

z.shape