<repo_name>BearCleverProud/Multilingual-Word-Embedding<file_sep>/create_language_pairs.py
import pickle
import torch
import LoadData
import string
import gensim

class Dict(object):

	def __init__(self):
		self.word2idx = {}
		self.idx2word = {}
		self.vocab_size = 0

def extract_words(en2frfile, fr2enfile):

	en_dict = {}
	fr_dict = {}
	threshold = 0.05

	with open(en2frfile, "r") as f:
		for line in f:
			fr, en, prob = line.split()
			if float(prob) > threshold:
				fr_dict[(fr, en)] = float(prob)

	print("Done loading FR to EN dictionary,", len(fr_dict.keys()), "pairs have been loaded")

	with open(fr2enfile, "r") as f:
		for line in f:
			en, fr, prob = line.split()
			if float(prob) > threshold:
				en_dict[(en, fr)] = float(prob)

	print("Done loading EN to FR dictionary,", len(en_dict.keys()), "pairs have been loaded")

	return en_dict, fr_dict

if __name__ == "__main__":
	# en_dict, fr_dict = extract_words("lex.1.e2f", "lex.1.f2e")
	# pickle.dump(en_dict, open("en_dict", "wb"))
	# print("Done dumped EN dictionary")
	# pickle.dump(fr_dict, open("fr_dict", "wb"))
	# print("Done dumped FR dictionary")
	
	en_dict = pickle.load(open("en_dict", "rb"))
	print("Done loaded en_dict")
	fr_dict = pickle.load(open("fr_dict", "rb"))
	print("Done loaded fr_dict")
	en_vocab = gensim.models.KeyedVectors.load_word2vec_format("../../polite-dialogue-generation/data/GoogleNews-vectors-negative300.bin", binary=True).vocab.keys()
	fr_vocab = torch.load("../code/vocab.fr")
	pairs = []
	for en, fr in en_dict.keys():
		if (len(en) == 1 and en in string.punctuation) or (len(fr) == 1 and fr in string.punctuation) or en == fr:
			continue
		if fr_dict.get((fr, en)) is not None and en in en_vocab and fr in fr_vocab.word2idx.keys() and en_dict[(en, fr)] * fr_dict[(fr, en)] > 0.1:
			print(en, fr, en_dict[(en, fr)], fr_dict[(fr, en)])
			pairs.append((en, fr))

	print(len(pairs), "of word pairs have been loaded.")
	pickle.dump(pairs, open("word_pairs", "wb"))
	print("Done dumped word pairs")





<file_sep>/evaluate.py
import torch

if __name__ == "__main__":

	word_pair_scores = {}
	cos = torch.nn.CosineSimilarity(dim=0, eps=1e-6)

	with open("SimLex-999.txt", "r") as f:
		for line in f:
			if not line.startswith("word1\tword2"):
				word1, word2, _, score, _, _, _, _, _, _ = line.split("\t")
				word_pair_scores[(word1, word2)] = float(score)

	vocab = torch.load("vocab")
	Word2Vec = torch.load("Word2Vec_CPU")
	correct = 0
	total = 0

	for word1, word2 in word_pair_scores:
		if word1 in vocab.word2idx and word2 in vocab.word2idx:
			total += 1
			similarity = cos(Word2Vec(torch.LongTensor([vocab.word2idx[word1]])).squeeze(), 
						Word2Vec(torch.LongTensor([vocab.word2idx[word2]])).squeeze())
			if abs(word_pair_scores[(word1, word2)] / 10 - similarity) <= 0.3:
				correct += 1


	print("Total Accuracy:", correct / total)


<file_sep>/README.md
1. First obtain the word embedding matrix of English and French according to the paper.
2. Run make_en.py and make_fr.py.
3. Get lex.1.e2f and lex.1.f2e from CLSP Grid, and put them in the same directory.
4. Uncomment the comment line of create_language_pairs.py, run the code.
5. Comment the comment line if you would like to speed it up for further use.
6. Run bilingual.py, you can check the argument, and maybe the code position needs to be modified



The rest of them are for training word embeddings, and evaluation of the result of word embedding.

1. Get ant corpus.
2. Run tokenization by Moses.
3. Run remove.py, which removes the name and number, replaced by <name>, <num>.
4. Run window.py, which takes consective 5 words to one line.
5. If you think the corpus is too large, you can always run shrink.py, and specify any size.
6. Then run train.py, there are arguments to be specified as bilingual.py.
7. If you have done training, you can run evaluation.py to evaluate the result.<file_sep>/window.py

w = open("small.fr", "w")
enough = False
count = 0
maximum = 1000000
with open("news.2018.fr.shuffled.tokenized.replace", "r") as f:

	for line in f:
		line_split = line.split()
		line_split.insert(0, "<bos>")
		line_split.insert(0, "<bos>")
		line_split.append("<eos>")
		line_split.append("<eos>")
		for i in range(2, len(line_split) - 2):
			new_line = line_split[i - 2] + " " + line_split[i - 1] + " " + line_split[i] + " " + line_split[i + 1] + " " + line_split[i + 2] + "\n"
			count += 1
			if count % 10000 == 0:
				print(count)
			if count == maximum:
				enough = True
				break
			else:
				w.write(new_line)
		if enough:
			break

w.close()<file_sep>/train.py
import torch.utils.data
import torch.optim
import torch.nn as nn
import torch
import LoadData
import Net
import argparse
import time
import numpy as np

start = time.time()
parser = argparse.ArgumentParser(description='train.py')
parser.add_argument('-epochs', default=100, type=int, help="number of epochs")
parser.add_argument('-batch_size', default=10000, type=int, help="batch size")
parser.add_argument('-embed_dimension', default=200, type=int, help="embedding dimension")
parser.add_argument('-gpus', default=[], nargs='+', type=int,
                    help="Use CUDA on the listed devices.")
parser.add_argument('-corpus', default="../data/small", type=str, help="corpus to train the model")
opt = parser.parse_args()
device = torch.device("cpu")
if len(opt.gpus) != 0:
    torch.cuda.set_device(opt.gpus[0])
    device = torch.device("cuda:" + str(opt.gpus[0]))

data_set = LoadData.DataSet(opt.corpus)
num_data = len(data_set)
print("Training instance:", num_data)
dl = torch.utils.data.DataLoader(data_set, batch_size=opt.batch_size, shuffle=True, num_workers=0)
criterion = nn.CrossEntropyLoss().to(device)
epochs = opt.epochs
network = Net.CBOWNet(20000, opt.embed_dimension).to(device)
print(device)
optimizer = torch.optim.Adam(network.parameters())
cos = torch.nn.CosineSimilarity(dim=0, eps=1e-6)
best_loss = 1000000.0

for i in range(epochs):
    network.train()
    steps = 0
    for idx, (data, labels) in enumerate(dl):
        data, labels = data.to(device), labels.to(device)
        steps += 1
        predict = network(data)
        loss = criterion(predict, labels)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        if steps == 1:
            print("Epoch:", i+1, "Steps:", steps, "PPL:", 2 ** loss.item())
        if loss.item() < best_loss:
            weight = list(network.embed.parameters())[0].data.cpu().numpy()
            new_weight = np.zeros_like(weight)
            for j in range(len(weight)):
                sum = np.sqrt(np.sum(np.square(weight[j])))
                new_weight[j] = weight[j] / sum
            network.embed.weight.data.copy_(torch.from_numpy(new_weight))
            best_loss = loss.item()
            torch.save(network.embed, "Word2Vec.fr")
            torch.save(network.embed.cpu(), "Word2Vec_CPU.fr")
            torch.save(network.embed.weight.cpu().detach().numpy(), "numpy_embed_test.fr")
            torch.save(data_set.dic, "vocab.fr")
            embed = torch.load("Word2Vec.fr")
            dic = torch.load("vocab.fr")
            king = torch.LongTensor([dic.word2idx["roi"]]).to(device)
            man = torch.LongTensor([dic.word2idx["homme"]]).to(device)
            woman = torch.LongTensor([dic.word2idx["femme"]]).to(device)
            queen = torch.LongTensor([dic.word2idx["reine"]]).to(device)
            king_minus_man_plus_woman = embed(king) - embed(man) + embed(woman)
            queen = embed(queen)
            output = "Cosine similarity between \"king + man - woman\" and \"queen\": {0:.3f}"
            print(output.format(cos(king_minus_man_plus_woman.squeeze(), queen.squeeze()).item()))
            paris = torch.LongTensor([dic.word2idx["paris"]]).to(device)
            france = torch.LongTensor([dic.word2idx["france"]]).to(device)
            uk = torch.LongTensor([dic.word2idx["royaume-uni"]]).to(device)
            london = torch.LongTensor([dic.word2idx["londres"]]).to(device)
            paris_minus_france_plus_london = embed(paris) - embed(france) + embed(uk)
            london = embed(london)
            output = "Cosine similarity between \"paris - france + uk\" and \"london\": {0:.3f}"
            print(output.format(cos(paris_minus_france_plus_london.squeeze(), london.squeeze()).item()))
            network = network.to(device)
<file_sep>/LoadData.py
from torch.utils.data import Dataset
import torch

class Dict(object):

    def __init__(self, corpus):
        self.idx2word, self.word2idx = build_dic(corpus)
        self.vocab_size = len(self.word2idx.keys())

    def to_idx(self, word):
        return self.word2idx[word]

    def to_word(self, idx):
        return self.idx2word[idx]

def build_dic(corpus):
    word_count = {}
    with open(corpus, "r") as f:
        for line in f:
            line_split = line.split()
            for word in line_split:
                if word_count.get(word) is None:
                    word_count[word] = 1
                else:
                    word_count[word] = word_count[word] + 1
    count = sorted(word_count.values(), reverse=True)
    print(len(count), "words have been loaded")
    if len(count) > 20000:
        threshold = count[19998]
        word_list = []
        for word in word_count.keys():
            if len(word_list) == 19997:
                break
            if word_count[word] >= threshold:
                word_list.append(word)
        words = word_list
    else:
        words = list(word_count.keys())
    words.append("<unk>")
    words.append("<BOS>")
    words.append("<EOS>")
    assert len(words) == 20000
    idx2word = {}
    word2idx = {}
    for i in range(len(words)):
        idx2word[i] = words[i]
        word2idx[words[i]] = i
    return idx2word, word2idx



class DataSet(Dataset):

    def __init__(self, corpus):
        self.data = open(corpus, 'r').read().strip().split("\n")
        for each in self.data:
            assert len(each.split()) == 5
        self.dic = Dict(corpus)
        self.data_size = len(self.data)

    def __len__(self):
        return len(self.data)

    def __getitem__(self, item):
        split = self.data[item].split()
        replace = []
        for each in split:
            replace.append(each) if each in self.dic.word2idx.keys() else replace.append("<unk>")
        tensor = torch.LongTensor([self.dic.word2idx[replace[0]], self.dic.word2idx[replace[1]],
                  self.dic.word2idx[replace[3]], self.dic.word2idx[replace[4]]])
        return tensor, self.dic.word2idx[replace[2]]


if __name__ == "__main__":

    dic = Dict("data/news.2018.en.shuffled.txt")
    print(dic.vocab_size)
    assert len(dic.word2idx.keys()) == len(dic.idx2word.keys())
<file_sep>/shrink.py
w = open("large.txt", "w")

maximum = 100000000
count = 0
with open("organised", "r") as f:

	for line in f:
		if line != "\n" and line != "":
			count += 1
			w.write(line)
			if count == maximum:
				break

w.close()

# w = open("large", "w")
# with open("large.txt", "r") as f:
# 	 for line in f:
# 	 	if line != "\n" and line != "":
# 	 		w.write(line)
# w.close()
<file_sep>/remove.py
from nltk.corpus import names

names=[name for name in names.words('male.txt')]+[name for name in names.words('female.txt')]

def digit(strings):
	if strings == "." or strings == ",":
		return False
	for each in strings:
		if each not in "0123456789,.":
			return False
	return True


w = open("news.2018.fr.shuffled.tokenized.replace","w")
with open("news.2018.fr.shuffled.tokenized", "r") as f:
	for line in f:
		result = ""
		scripts = line.split(" ")
		for script in scripts:
			if script in names:
				result += "<person> "
			elif digit(script):
				result += "<number> "
			else:
				result += script + " "
		w.write(result.lower())
w.close()<file_sep>/Net.py

import torch.nn as nn
import torch


class CBOWNet(nn.Module):

    def __init__(self, vocab_size, dimension):

        super(CBOWNet, self).__init__()
        self.embed = nn.Embedding(vocab_size, dimension)
        self.fc1 = nn.Linear(dimension, vocab_size)
        # self.argmax = torch.argmax(dim=1)
        # self.softmax = nn.LogSoftmax(dim=1)
        # self.fc1 = nn.Linear(vocab_size, dimension)
        # self.fc2 = nn.Linear(dimension, vocab_size)

    def forward(self, x):
        x = self.embed(x)
        x = x.mean(dim=1)
        # x = self.argmax(x)
        x = self.fc1(x)
        # x = self.softmax(x)
        return x


class SkipGramNet(nn.Module):

    def __init__(self, vocab_size, dimension):

        super(SkipGramNet, self).__init__()
        self.fc1 = nn.Linear(vocab_size, dimension)
        self.softmax = nn.Softmax(dim=0)

    def forward(self, x):

        x = self.fc1(x)
        x = self.softmax(x)
        return x
<file_sep>/bilingual.py
import torch.utils.data
import torch.optim
import torch.nn as nn
import torch
import argparse
import time
import pickle
import numpy as np
# from scipy.spatial.distance import cdist
import gensim

class Dict(object):

	def __init__(self):
		self.word2idx = {}
		self.idx2word = {}
		self.vocab_size = 0

class Bilingual(nn.Module):

    def __init__(self, en_embed, fr_embed):

        super(Bilingual, self).__init__()
        self.fc = torch.nn.Linear(en_embed, fr_embed)
        self.dropout = torch.nn.Dropout(p=0.4)

    def forward(self, x):
        x = self.fc(x)
        x = self.dropout(x)
        return x


def evaluate(en_embed, fr_embed, network, device):
    total = len(en_test_words)
    correct = 0
    net.eval()

    en_vecs = en_embed[torch.LongTensor(en_test_idxs)].to(device).squeeze()
    en_vecs = network(en_vecs)
    # en_vecs = [en_model[w] for w in en_test_words]
    # en_vecs = torch.FloatTensor(en_vecs).to(device).squeeze()
    # en_vecs = network(en_vecs).squeeze()
    fr_vecs = fr_embed[torch.LongTensor(fr_test_idxs)].to(device).squeeze()
    cos = torch.nn.CosineSimilarity(dim=0)
    highest = -2
    idx = -2
    print("Start to evaluate")
    for j in range(len(en_vecs)):
        for k in range(len(fr_vecs)):
            result = cos(en_vecs[j], fr_vecs[k])
            if result > highest:
                highest = result
                idx = k
        if idx == j:
            print(j, str(correct) + "th correct")
            correct += 1
        highest = -2
        idx = -2


    # distance = cdist(en_vecs, fr_vecs, metric='cosine')
    # distance = np.argmax(distance, axis=0)
    # for j in range(len(distance)):
    #     print(j, distance[j], en_test_words[j], fr_test_words[j], cos(torch.Tensor(en_vecs[j]).squeeze(), torch.Tensor(fr_vecs[j]).squeeze()))
    #     print(j, distance[j], en_test_words[j], fr_test_words[distance[j]], cos(torch.Tensor(en_vecs[j]).squeeze(), torch.Tensor(fr_vecs[distance[j]]).squeeze()))
    #     if distance[j] == j:
    #         correct += 1

    #
    # predict = network(en_vec)
    # similarity = torch.sum(predict.mul(fr_vec), dim=1) / torch.sqrt(torch.sum(predict**2, dim=1))
    # for each in similarity:
    #     if each > 0.5:
    #         correct += 1
    print("Good result percentage:", correct / total)

def orthogonal(matrix):
    u, s, vh = np.linalg.svd(matrix)
    return u.dot(np.eye(matrix.shape[0])).dot(vh.conjugate())


start = time.time()
parser = argparse.ArgumentParser(description='train.py')
parser.add_argument('-epochs', default=500, type=int, help="number of epochs")
parser.add_argument('-gpus', default=[], nargs='+', type=int,
                    help="Use CUDA on the listed devices.")
parser.add_argument('-en_dimension', default=300, type=int, help="embedding dimension of English")
parser.add_argument('-fr_dimension', default=300, type=int, help="embedding dimension of French")
parser.add_argument('-partition_ratio', default=0.7, type=float, help="ratio to partition the data")
en_dic = torch.load("en_vocab")
fr_dic = torch.load("vocab.fr")
language_pairs = pickle.load(open("word_pairs", "rb"))
opt = parser.parse_args()
device = torch.device("cpu")
if len(opt.gpus) != 0:
    torch.cuda.set_device(opt.gpus[0])
    device = torch.device("cuda:" + str(opt.gpus[0]))

en_embed = np.array([torch.load("en_vector_part_1"), torch.load("en_vector_part_2"), torch.load("en_vector_part_3")])
new_en = torch.nn.Embedding(len(en_embed) * en_embed.shape[1], opt.en_dimension)
en_embed = new_en.weight.data.copy_(torch.from_numpy(en_embed).view(-1, opt.en_dimension)).to(device)
# en_model = gensim.models.KeyedVectors.load_word2vec_format('GoogleNews-vectors-negative300.bin', binary=True)

fr_embed = torch.FloatTensor(torch.load("Word2Vec_CPU.fr")).to(device)

net = Bilingual(opt.en_dimension, opt.fr_dimension).to(device)
criterion = nn.CosineEmbeddingLoss().to(device)
optimizer = torch.optim.Adam(net.parameters())
en_idxs = [en_dic.word2idx[en] for en, _ in language_pairs]
fr_idxs = [fr_dic.word2idx[fr] for _, fr in language_pairs]
assert len(en_idxs) == len(fr_idxs)
en_words = [en for en, _ in language_pairs]
data_len = len(fr_idxs)
print("total data number:", data_len)
en_train_words = en_words[0: int(opt.partition_ratio * data_len)]
en_train_idxs = [en_dic.word2idx[each] for each in en_train_words]
en_test_words = en_words[int(0.7 * data_len): int(1 * data_len)]
en_test_idxs = [en_dic.word2idx[each] for each in en_test_words]
fr_train_idxs = fr_idxs[0: int(opt.partition_ratio * data_len)]
fr_test_idxs = fr_idxs[int(0.7 * data_len): int(1 * data_len)]
# en_test_words = [en_dic.idx2word[each] for each in en_test_idxs]
fr_test_words = [fr_dic.idx2word[each] for each in fr_test_idxs]
# en_train_words = [en_dic.idx2word[each] for each in en_train_idxs]
fr_train_words = [fr_dic.idx2word[each] for each in fr_train_idxs]

net.train()
# en_vec = en_embed[torch.LongTensor(en_train_idxs)].to(device).squeeze()
fr_vec = fr_embed[torch.LongTensor(fr_train_idxs)].to(device).squeeze()
en_vec = en_embed[torch.LongTensor(en_train_idxs)].to(device).squeeze()
# en_vec = [en_model[w] for w in en_train_words]
# en_vec = torch.FloatTensor(en_vec).to(device).squeeze()

for i in range(opt.epochs):
    pred = net(en_vec)
    dot = fr_vec.mul(pred)
    loss = - torch.sum(torch.sum(dot, dim=1) / (torch.sqrt(torch.sum(pred ** 2, dim=1))) / (torch.sqrt(torch.sum(fr_vec ** 2, dim=1))))
    # loss = criterion(pred, fr_vec, target=torch.ones(pred.shape[0]))

    optimizer.zero_grad()
    loss.backward()
    optimizer.step()
    print(i + 1, "epoch, WXZ:", - loss.item())

list(net.parameters())[0].data.copy_(torch.Tensor(orthogonal(list(net.parameters())[0].data.detach().cpu().numpy()))).to(device)
print("Done orthogonal")
evaluate(en_embed, fr_embed, net, device)

torch.save(net, "bilingual.torch")


<file_sep>/make_en.py
import gensim
import torch
import numpy as np

class Dict(object):

    def __init__(self):
        self.word2idx = {}
        self.idx2word = {}
        self.vocab_size = 0

en_model = gensim.models.KeyedVectors.load_word2vec_format('../../polite-dialogue-generation/data/GoogleNews-vectors-negative300.bin', binary=True)
dict_ = Dict()
vector = []
i = 0
vocab_size = len(en_model.vocab.keys())
for each in en_model.vocab.keys():
	vector.append(en_model[each] / np.sqrt(np.sum(en_model[each] ** 2)))
	dict_.word2idx[each] = i
	dict_.idx2word[i] = each
	i += 1
	if i == vocab_size // 3:
		torch.save(np.array(vector), "en_vector_part_1", pickle_protocol=4)
		vector = []
	elif i == 2 * vocab_size // 3:
		torch.save(np.array(vector), "en_vector_part_2", pickle_protocol=4)
		vector = []
dict_vocab_size = i
torch.save(dict_, "en_vocab")
torch.save(np.array(vector), "en_vector_part_3", pickle_protocol=4)
<file_sep>/requirements.txt
numpy=1.15.4
torch=1.0.1.post2
gensim=3.4.0<file_sep>/source.py
import re
import nltk
nltk.download('brown')
from nltk.corpus import brown
import itertools
import argparse

corpus = []

parser = argparse.ArgumentParser(description='train.py')
parser.add_argument('-batch_size', default=10000, type=int, help="batch size")
parser.add_argument('-embed_dimension', default=100, type=int, help="embedding dimension")
parser.add_argument('-gpu', type=int, help="Use CUDA on the listed devices.")
opt = parser.parse_args()

for cat in ['news']:
    for text_id in brown.fileids(cat):
        raw_text = list(itertools.chain.from_iterable(brown.sents(text_id)))
        text = ' '.join(raw_text)
        text = text.lower()
        text.replace('\n', ' ')
        text = re.sub('[^a-z ]+', '', text)
        corpus.append([w for w in text.split() if w != ''])

from collections import Counter
import random, math

def subsample_frequent_words(corpus):
    filtered_corpus = []
    word_counts = dict(Counter(list(itertools.chain.from_iterable(corpus))))
    sum_word_counts = sum(list(word_counts.values()))
    word_counts = {word: word_counts[word]/float(sum_word_counts) for word in word_counts}
    for text in corpus:
        filtered_corpus.append([])
        for word in text:
            if random.random() < (1+math.sqrt(word_counts[word] * 1e3)) * 1e-3 / float(word_counts[word]):
                filtered_corpus[-1].append(word)
    return filtered_corpus

corpus = subsample_frequent_words(corpus)
vocabulary = set(itertools.chain.from_iterable(corpus))

word_to_index = {w: idx for (idx, w) in enumerate(vocabulary)}
index_to_word = {idx: w for (idx, w) in enumerate(vocabulary)}


import numpy as np

context_tuple_list = []
w = 4

for text in corpus:
    for i, word in enumerate(text):
        first_context_word_index = max(0,i-w)
        last_context_word_index = min(i+w, len(text))
        for j in range(first_context_word_index, last_context_word_index):
            if i!=j:
                context_tuple_list.append((word, text[j]))
print("There are {} pairs of target and context words".format(len(context_tuple_list)))

import torch
import torch.nn  as  nn
import torch.autograd  as autograd
import torch.optim as optim
import torch.nn.functional as F

device = torch.device("cpu")
if opt.gpu is not None:
	device = torch.device("cuda:" + str(opt.gpu))


class Word2Vec(nn.Module):

    def __init__(self, embedding_size, vocab_size):
        super(Word2Vec, self).__init__()
        self.embeddings = nn.Embedding(vocab_size, embedding_size)
        self.linear = nn.Linear(embedding_size, vocab_size)
        
    def forward(self, context_word):
        emb = self.embeddings(context_word)
        hidden = self.linear(emb)
        out = F.log_softmax(hidden)
        return out

class EarlyStopping():
    def __init__(self, patience=5, min_percent_gain=0.1):
        self.patience = patience
        self.loss_list = []
        self.min_percent_gain = min_percent_gain / 100.
        
    def update_loss(self, loss):
        self.loss_list.append(loss)
        if len(self.loss_list) > self.patience:
            del self.loss_list[0]
    
    def stop_training(self):
        if len(self.loss_list) == 1:
            return False
        gain = (max(self.loss_list) - min(self.loss_list)) / max(self.loss_list)
        print("Loss gain: {}%".format(round(100*gain,2)))
        if gain < self.min_percent_gain:
            return True
        else:
            return False

vocabulary_size = len(vocabulary)

net = Word2Vec(embedding_size=2, vocab_size=vocabulary_size)
loss_function = nn.CrossEntropyLoss()
optimizer = optim.Adam(net.parameters())
early_stopping = EarlyStopping()
context_tensor_list = []

for target, context in context_tuple_list:
    target_tensor = autograd.Variable(torch.LongTensor([word_to_index[target]]))
    context_tensor = autograd.Variable(torch.LongTensor([word_to_index[context]]))
    context_tensor_list.append((target_tensor, context_tensor))
    
while True:
    losses = []
    for target_tensor, context_tensor in context_tensor_list:
        net.zero_grad()
        log_probs = net(context_tensor)
        loss = loss_function(log_probs, target_tensor)
        loss.backward()
        optimizer.step()
        losses.append(loss.data)
    print("Loss: ", np.mean(losses))
    early_stopping.update_loss(np.mean(losses))
    if early_stopping.stop_training():
        break
import random

def get_batches(context_tuple_list, batch_size=100):
    random.shuffle(context_tuple_list)
    batches = []
    batch_target, batch_context, batch_negative = [], [], []
    for i in range(len(context_tuple_list)):
        batch_target.append(word_to_index[context_tuple_list[i][0]])
        batch_context.append(word_to_index[context_tuple_list[i][1]])
        batch_negative.append([word_to_index[w] for w in context_tuple_list[i][2]])
        if (i+1) % batch_size == 0 or i == len(context_tuple_list)-1:
            tensor_target = autograd.Variable(torch.from_numpy(np.array(batch_target)).long())
            tensor_context = autograd.Variable(torch.from_numpy(np.array(batch_context)).long())
            tensor_negative = autograd.Variable(torch.from_numpy(np.array(batch_negative)).long())
            batches.append((tensor_target, tensor_context, tensor_negative))
            batch_target, batch_context, batch_negative = [], [], []
    return batches

from numpy.random import multinomial

def sample_negative(sample_size):
    sample_probability = {}
    word_counts = dict(Counter(list(itertools.chain.from_iterable(corpus))))
    normalizing_factor = sum([v**0.75 for v in word_counts.values()])
    for word in word_counts:
        sample_probability[word] = word_counts[word]**0.75 / normalizing_factor
    words = np.array(list(word_counts.keys()))
    while True:
        word_list = []
        sampled_index = np.array(multinomial(sample_size, list(sample_probability.values())))
        for index, count in enumerate(sampled_index):
            for _ in range(count):
                 word_list.append(words[index])
        yield word_list

import numpy as np

context_tuple_list = []
w = 4
negative_samples = sample_negative(8)

for text in corpus:
    for i, word in enumerate(text):
        first_context_word_index = max(0,i-w)
        last_context_word_index = min(i+w, len(text))
        for j in range(first_context_word_index, last_context_word_index):
            if i!=j:
                context_tuple_list.append((word, text[j], next(negative_samples)))
print("There are {} pairs of target and context words".format(len(context_tuple_list)))

import torch
import torch.nn as nn
import torch.autograd as autograd
import torch.optim as optim
import torch.nn.functional as F


class Word2Vec(nn.Module):

    def __init__(self, embedding_size, vocab_size):
        super(Word2Vec, self).__init__()
        self.embeddings_target = nn.Embedding(vocab_size, embedding_size)
        self.embeddings_context = nn.Embedding(vocab_size, embedding_size)

    def forward(self, target_word, context_word, negative_example):
        emb_target = self.embeddings_target(target_word)
        emb_context = self.embeddings_context(context_word)
        emb_product = torch.mul(emb_target, emb_context)
        emb_product = torch.sum(emb_product, dim=1)
        out = torch.sum(F.logsigmoid(emb_product))
        emb_negative = self.embeddings_context(negative_example)
        emb_product = torch.bmm(emb_negative, emb_target.unsqueeze(2))
        emb_product = torch.sum(emb_product, dim=1)
        out += torch.sum(F.logsigmoid(-emb_product))
        return -out

import time

vocabulary_size = len(vocabulary)

loss_function = nn.CrossEntropyLoss().to(device)
net = Word2Vec(embedding_size=opt.embedding_size, vocab_size=vocabulary_size).to(device)
optimizer = optim.Adam(net.parameters())
early_stopping = EarlyStopping(patience=5, min_percent_gain=1)

while True:
    losses = []
    context_tuple_batches = get_batches(context_tuple_list, batch_size=opt.batch_size)
    for i in range(len(context_tuple_batches)):
        net.zero_grad()
        target_tensor, context_tensor, negative_tensor = context_tuple_batches[i]
        target_tensor, context_tensor, negative_tensor = target_tensor.to(device), context_tensor.to(device), negative_tensor.to(device)
        loss = net(target_tensor, context_tensor, negative_tensor)
        loss.backward()
        optimizer.step()
        losses.append(loss.data)
    print("Loss: ", np.mean(losses))
    early_stopping.update_loss(np.mean(losses))
    if early_stopping.stop_training():
        break


import numpy as np

def get_closest_word(word, topn=5):
    word_distance = []
    emb = net.embeddings_target
    pdist = nn.PairwiseDistance()
    i = word_to_index[word]
    lookup_tensor_i = torch.tensor([i], dtype=torch.long)
    v_i = emb(lookup_tensor_i)
    for j in range(len(vocabulary)):
        if j != i:
            lookup_tensor_j = torch.tensor([j], dtype=torch.long)
            v_j = emb(lookup_tensor_j)
            word_distance.append((index_to_word[j], float(pdist(v_i, v_j))))
    word_distance.sort(key=lambda x: x[1])
    return word_distance[:topn]<file_sep>/make_fr.py
import io
import torch
import numpy as np

class Dict(object):

    def __init__(self):
        self.word2idx = {}
        self.idx2word = {}
        self.vocab_size = 0

def load_vectors(fname):
    fin = io.open(fname, 'r', encoding='utf-8', newline='\n', errors='ignore')
    vocab = Dict()
    vector = []
    i = 0
    for line in fin:
        if i == 0:
            i += 1
            continue
        tokens = line.rstrip().split(' ')
        fr, vecs = tokens[0], tokens[1:]
        vec = np.array([float(component) for component in vecs])
        vec /= np.sqrt(np.sum(vec ** 2))
        vec = vec.tolist()
        vector.append(vec)
        vocab.word2idx[fr] = i - 1
        vocab.idx2word[i - 1] = fr
        vocab.vocab_size = i
        i += 1
    return vocab, vector

if __name__ == '__main__':
    vocab, vector = load_vectors("cc.fr.300.vec")
    vector = torch.save(torch.Tensor(vector), "Word2Vec_CPU.fr")
    torch.save(vocab, "vocab.fr")
