image_output_1.png
from IPython.display import FileLink

from six.moves import xrange

import collections

import math

import numpy as np

import os

import random

import tensorflow as tf

import tensorflow.compat.v1 as tf

folder_dir = '/kaggle/input/108-2-ntut-dl-app-hw2'
filename = 'tag_list.txt'
vocabulary_size = 990

batch_size = 128
embedding_size = 64
skip_window = 1
num_skips = 2

valid_size = 32
valid_window = 200
valid_examples = np.random.choice(valid_window, valid_size, replace=False)

file_path = os.path.join(folder_dir, filename)
with open(file_path, 'r', encoding='utf-8') as f:
    words = f.read().split()

word_count = [['UNK', -1]]
word_count.extend(collections.Counter(words).most_common(vocabulary_size - 1))

dictionary = dict()
for word, _ in word_count:
    dictionary[word] = len(dictionary)
reverse_dictionary = dict(zip(dictionary.values(), dictionary.keys()))

data = list()
unk_count = 0
for word in words:
    if word in dictionary:
        index = dictionary[word]
    else:
        index = 0
        unk_count += 1
    data.append(index)
word_count[0][1] = unk_count

data_index = 0
def generate_batch(batch_size, num_skips, skip_window):
    global data_index
    assert batch_size % num_skips == 0
    assert num_skips <= 2 * skip_window
    batch = np.ndarray(shape=batch_size, dtype=np.int32)
    labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)
    span = 2 * skip_window + 1
    buffer = collections.deque(maxlen=span)
    for _ in range(span):
        buffer.append(data[data_index])
        data_index = (data_index + 1) % len(data)
    for i in range(batch_size // num_skips):
        target = skip_window
        targets_to_avoid = [skip_window]
        for j in range(num_skips):
            while target in targets_to_avoid:
                target = random.randint(0, span - 1)
            targets_to_avoid.append(target)
            batch[i * num_skips + j] = buffer[skip_window]
            labels[i * num_skips + j, 0] = buffer[target]
        buffer.append(data[data_index])
        data_index = (data_index + 1) % len(data)
    return (batch, labels)

train_inputs = tf.placeholder(tf.int32, shape=[batch_size])
train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])
valid_dataset = tf.constant(valid_examples, dtype=tf.int32)
with tf.variable_scope('EMBEDDING'):
    with tf.device('/cpu:0'):
        embeddings = tf.Variable(tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))
        embed = tf.nn.embedding_lookup(embeddings, train_inputs)
with tf.variable_scope('NCE_WEIGHT'):
    nce_weights = tf.Variable(tf.truncated_normal([vocabulary_size, embedding_size], stddev=1.0 / math.sqrt(embedding_size)))
    nce_biases = tf.Variable(tf.zeros([vocabulary_size]))

with tf.device('/cpu:0'):
    num_sampled = 64
    loss = tf.reduce_mean(tf.nn.nce_loss(weights=nce_weights, biases=nce_biases, labels=train_labels, inputs=embed, num_sampled=num_sampled, num_classes=vocabulary_size))
    optm = tf.train.GradientDescentOptimizer(1.0).minimize(loss)
    norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
    normalized_embeddings = embeddings / norm
    valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings, valid_dataset)
    siml = tf.matmul(valid_embeddings, normalized_embeddings, transpose_b=True)

sess = tf.Session()
sess.run(tf.initialize_all_variables())
average_loss = 0
num_steps = 10001
for iter in xrange(num_steps):
    batch_inputs, batch_labels = generate_batch(batch_size, num_skips, skip_window)
    feed_dict = {train_inputs: batch_inputs, train_labels: batch_labels}
    _, loss_val = sess.run([optm, loss], feed_dict=feed_dict)
    average_loss += loss_val
    if iter % 2000 == 0:
        average_loss /= 2000
    if iter % 10000 == 0:
        siml_val = sess.run(siml)
        for i in xrange(valid_size):
            valid_word = reverse_dictionary[valid_examples[i]]
            top_k = 6
            nearest = (-siml_val[i, :]).argsort()[1:top_k + 1]
            log_str = "Nearest to '%s':" % valid_word
            for k in xrange(top_k):
                close_word = reverse_dictionary[nearest[k]]
                log_str = "%s '%s'," % (log_str, close_word)
final_embeddings = sess.run(normalized_embeddings)

np.savez(filename[0:-4] + '_word2vec_' + str(embedding_size), word_count=word_count, dictionary=dictionary, reverse_dictionary=reverse_dictionary, word_embeddings=final_embeddings)

K = 10
target = 'drunk'
scores = final_embeddings[dictionary[target]].dot(final_embeddings.transpose())
scores = scores / np.linalg.norm(final_embeddings, axis=1)
k_neighbors = (-scores).argsort()[0:K + 1]

out_v = open('vecs.tsv', 'w', encoding='utf-8')
out_m = open('meta.tsv', 'w', encoding='utf-8')
for num, word in enumerate(dictionary):
    vec = final_embeddings[num]
    out_m.write(word + '\n')
    out_v.write('\t'.join([str(x) for x in vec]) + '\n')
out_v.close()
out_m.close()
from IPython.display import FileLink
FileLink('vecs.tsv')