application_vnd.jupyter.stderr_output_1.png image_output_1.png
from gensim.models import KeyedVectors, Word2Vec

from matplotlib import pyplot

from nltk.tokenize import RegexpTokenizer

from sklearn.decomposition import PCA

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

forum_posts = pd.read_csv('../input/meta-kaggle/ForumMessages.csv')['Message'].astype('str')

tokenizer = RegexpTokenizer('\\w+')
data_tokenized = [w.lower() for w in forum_posts.tolist()]
data_tokenized = [tokenizer.tokenize(i) for i in data_tokenized]

model_2 = Word2Vec(size=300, min_count=1)
model_2.build_vocab(data_tokenized)
total_examples = model_2.corpus_count
model_2.train(data_tokenized, total_examples=total_examples, epochs=1)

X = model_2[list(model_2.wv.vocab.keys())[:100]]
pca = PCA(n_components=2)
result = pca.fit_transform(X)
pyplot.scatter(result[:, 0], result[:, 1])
words = list(model_2.wv.vocab.keys())[:100]
for i, word in enumerate(words):
    pyplot.annotate(word, xy=(result[i, 0], result[i, 1]))
pyplot.show()