<file_sep>
#include <stdio.h>
#include "hash_mapping_table.h"

void test_mapping_table_1() {
    int result;

    result = has_common_character("国王", "国家", 0);
    fprintf(stderr, "result = %d\n", result);

    result = has_common_character("国王", "国家", 1);
    fprintf(stderr, "result = %d\n", result);

}

// void test_mapping_table_2() {
//     int result;

//     result = BinarySearchTable(sc2k, sc2k[sc2k_size - 1]->original, 0, sc2k_size - 1);
//     fprintf(stderr, "result = %d (%d)\n", result, sc2k_size - 1);

//     result = BinarySearchTable(k2sc, k2sc[k2sc_size - 1]->original, 0, k2sc_size - 1);
//     fprintf(stderr, "result = %d (%d)\n", result, k2sc_size - 1);
// }

// void test() {
//     test_mapping_table_1(0);
//     test_mapping_table_1(7);
//     test_mapping_table_1(77);
//     test_mapping_table_1(777);
//     test_mapping_table_1(5000);
//     test_mapping_table_2();
// }

int main(int argc, char **argv) {
    load_mapping_tables();
    fprintf(stderr, "Finish reading from file.\n");
    // test();
    test_mapping_table_1();
}<file_sep>//  Tool to calculate word-word cooccurrence statistics for sentence aligned
//  corpus of 2 languages.
//
//  Copyright (c) 2014 The Board of Trustees of
//  The Leland Stanford Junior University. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
//
//
//  For more information, bug reports, fixes, contact:
//    <NAME> (<EMAIL>)


#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include "bisearch_mapping_table.h"


MAPTABREC **k2sc, **sc2k;

/**
 * Compare 2 mapping table record, used in qsort function.
 * notice the type of a and b are (MAPTABREC **)
 */
int compare_original(const void *a, const void *b) {
    return strcmp((*(MAPTABREC **)a)->original, (*(MAPTABREC **)b)->original);
}

/**
 * Read one or several Kanji or Hanzi from text file
 */
int get_characters(char *word, FILE *fin) {
    int i = 0, ch;
    while (!feof(fin)) {
        ch = fgetc(fin);
        if ((ch == ' ') || (ch == '\t') || (ch == '\n')) {
            break;
        }
        word[i++] = ch;
    }
    word[i] = 0;
    return i;
}

/**
 * Read mapping table from file, and sort it
 */
MAPTABREC ** load_one_mapping_table(char *filename, int table_size) {
    int i;
    FILE *f;
    char *original, *corresponding;
    MAPTABREC *table_record;
    MAPTABREC **table;
    table = (MAPTABREC **)malloc( sizeof(MAPTABREC *) * table_size );

    f = fopen(filename, "rb");
    for (i = 0; i < table_size; i++) {      
        table[i] = (MAPTABREC *)malloc( sizeof(MAPTABREC) );
        get_characters(table[i]->original, f);
        get_characters(table[i]->corresponding, f);
    }
    fclose(f);

    qsort(&table[0], table_size, sizeof(MAPTABREC*), compare_original);

    return table;
}

/**
 * Read 2 mapping tables
 */
void load_mapping_tables() {
    sc2k = load_one_mapping_table(SC2K_FILENAME, SC2K_SIZE);
    k2sc = load_one_mapping_table(K2SC_FILENAME, K2SC_SIZE);
}

/**
 * Binary Search a character from the mapping table
 */
int bisearch_table(MAPTABREC **table, char *ch, int left, int right) {
    int mid, cmp;
    if (right < left) {
        return -1;
    }
    mid = left + (right - left) / 2;
    
    cmp = strcmp(table[mid]->original, ch);

    if (cmp == 0) {
        return mid;
    }
    else if (cmp > 0) {
        return bisearch_table(table, ch, left, mid - 1);
    }
    else {
        return bisearch_table(table, ch, mid + 1, right);
    }
}

int compare_characters(char *corresponding, char *word2) {
  int len1, len2, i, j;
  len1 = strlen(corresponding);
  len2 = strlen(word2);
  for (i = 0; i < len1; i+= 3) {
    for (j = 0; j < len2; j += 3) {
      if (word2[j] == corresponding[i] && word2[j+1] == corresponding[i+1] && word2[j+2] == corresponding[i+2]) {
        //if (verbose > 2) fprintf(stderr, "source_ch %s, target_ch %s, j %d, word2 %s\n", source_ch, target_ch, j, word2);
        return 1;
      }
    }
  }
  return 0;
}

/**
 * return 1 if the 2 words has a common character.
 */
int has_common_character(char *word1, char *word2, int lang_id) {
  char source_ch[4], target_ch[4];
  int len1, len2;
  int i, j, pos, table_size;
  int parallel = 1 - lang_id;
  MAPTABREC **table;

  if (lang_id == 1) {
    table = sc2k;
    table_size = SC2K_SIZE;
  }
  else {
    table = k2sc;
    table_size = K2SC_SIZE;
  }

  len1 = strlen(word1);
  len2 = strlen(word2);
  if (len1 % 3 != 0 || len2 % 3 != 0) {
    return 0;
  }
  // if (verbose > 2) fprintf(stderr, "CJWordMatch : %s - %s - %d\n", word1, word2, lang_id);
  source_ch[3] = 0;
  target_ch[3] = 0;
  for (i = 0; i < len1; i += 3) {
    // get a character from word1
    for (j = 0; j < 3; j++) {
      source_ch[j] = word1[i + j];
    }
    pos = bisearch_table(table, source_ch, 0, table_size - 1);
    if (pos >= 0) {
        if (compare_characters(table[pos]->corresponding, word2) > 0) {
            return 1;
        }
    }
  }
  return 0;
}



<file_sep>from collections import defaultdict
import codecs

def count(corpus, output_file):
    debug = False
    dic = defaultdict(int)
    other = set()
    fout = codecs.open(output_file, 'w', 'utf8')
    for line in open(corpus, 'r'):
        words = line.split()
        for word in words:
            if len(word) % 3 == 0:
                for i in xrange(len(word) / 3):
                    dic[word[i:i+3]] += 1
            else:
                other.add(word)
    fout.write('%i %i\n' % (len(dic), len(other)))
    
    record_list = [(y, x) for x, y in dic.items()]
    record_list.sort()
    record_list.reverse()
    i = 0
    for x, y in record_list:
        #print y.decode('utf8'), x
        try:
            yy = y.decode('GBK')
        except:
            print y
            yy = 'N/A'
        fout.write('%s %i\n' % (yy, x))
        i += 1
        if i > 10 and debug:
            break

    other_list = list(other)
    other_list.sort()
    for item in other_list:
        #print item.decode('utf8')
        item2 = item.decode('utf8')
        fout.write(item2)
        fout.write('\n')
        i += 1
        if i > 20 and debug:
            break
    fout.close()

if __name__ =='__main__':
    count('data/train.zh_parsed', 'output/count.zh')
    count('data/train.ja_parsed', 'output/count.ja')
<file_sep>"""
    Experiment 2
    Use a file of word pairs, to calculate Cross-Lingual Synonym Comparison 
"""

import argparse
import numpy as np
import sys

class WordVecDict(object):
    def __init__(self, vocab_file, vectors_file):
        self.vocab_file = vocab_file
        self.vectors_file = vectors_file
        self.generate()

    def generate(self):
        with open(self.vocab_file, 'r') as f:
            words = [x.rstrip().split(' ')[0] for x in f.readlines()]
        with open(self.vectors_file, 'r') as f:
            vectors = {}
            for line in f:
                vals = line.rstrip().split(' ')
                vectors[vals[0]] = [float(x) for x in vals[1:]]

        vocab_size = len(words)
        vocab = {w: idx for idx, w in enumerate(words)}
        ivocab = {idx: w for idx, w in enumerate(words)}

        vector_dim = len(vectors[ivocab[0]])
        W = np.zeros((vocab_size, vector_dim))
        for word, v in vectors.items():
            if word == '<unk>' or word == '2</s>':
                continue
            W[vocab[word], :] = v

        # normalize each word vector to unit variance
        W_norm = np.zeros(W.shape)
        d = (np.sum(W ** 2, 1) ** (0.5))
        W_norm = (W.T / d).T
        
        self.W = W_norm
        self.vocab = vocab
        self.ivocab = ivocab

    def distance(self, word1, word2, verbose = False):
        if not word1 in self.vocab:
            print('Word: %s  Out of dictionary!\n' % word1)
            return
        if not word2 in self.vocab:
            print('Word: %s  Out of dictionary!\n' % word2)
            return
        vec_result = np.copy(self.W[self.vocab[word1], :])
        
        vec_norm = np.zeros(vec_result.shape)
        d = (np.sum(vec_result ** 2,) ** (0.5))
        vec_norm = (vec_result.T / d).T

        dist = np.dot(self.W, vec_norm.T)

        #for term in input_term.split(' '):
        index = self.vocab[word1]
        dist[index] = -np.Inf

        a = np.argsort(-dist)#[:N]

        idx = 0
        for x in a:
            # in same language
            if self.ivocab[x][0] == word2[0]:
                idx += 1
                if verbose:
                    print idx, self.ivocab[x], dist[x]
                if self.ivocab[x] == word2:
                    return idx, dist[x]
        print("Error: %s -> %s %f" % (word1, word2))
        return -1, -1

    def analogy(self, words, verbose = False):
        """
            words = <w0, w1, w2, w3>
            Find distance of (w1 - w0 + w2) and w3
        """
        v0 = np.copy(self.W[self.vocab[words[0]], :])
        v1 = np.copy(self.W[self.vocab[words[1]], :])
        v2 = np.copy(self.W[self.vocab[words[2]], :])
        v = np.add(v1, v2)
        vec_result = np.subtract(v, v0)

        vec_norm = np.zeros(vec_result.shape)
        d = (np.sum(vec_result ** 2,) ** (0.5))
        vec_norm = (vec_result.T / d).T

        dist = np.dot(self.W, vec_norm.T)

        #for term in input_term.split(' '):
        for i in [0, 1, 2]:
            index = self.vocab[words[i]]
            dist[index] = -np.Inf

        a = np.argsort(-dist)#[:N]

        idx = 0
        for x in a:
            # in same language
            if self.ivocab[x][0] == words[3][0]:
                idx += 1
                if verbose:
                    print idx, self.ivocab[x], dist[x]
                if self.ivocab[x] == words[3]:
                    return idx, dist[x]
        print("Error")
        return -1, -1



    def sentenceVec(self, sentence):
        """
            input sentence = <w0, w1, ...>
            calculate the average data of corresponding <v0, v1, v2>
        """
        sen = []
        for w in sentence:
            if w in self.vocab:
                sen.append(w)

        if len(sen) == 0:
            return None

        tmp = self.W[0, :]
        vec = np.zeros(tmp.shape)
        for w in sen:
            v = np.copy(self.W[self.vocab[w], :])
            vec = np.add(vec, v)
        vec = vec / (len(sen) * 1.0)
        return vec





def readWordPairs():
    path = "experiment/experiment2_data.txt"
    fin = open(path, 'r')
    wordPairList = []
    for line in fin.readlines():
        if len(line) > 2:
            words = line.split()
            w1 = "2" + words[0].strip()
            w2 = "1" + words[1].strip()
            wordPairList.append((w1, w2))
    return wordPairList

if __name__ == "__main__":
    print "reading word embeddings"
    wordVecDict = WordVecDict("output/vocab.cj_001", "output/batch_02/ctrl_s_0.03.vec.txt")
    print "reading word pairs"
    wordPairList = readWordPairs()
    print "calculating distances"

    ja_wc = 22748
    zh_wc = 25219

    result = []
    # ja, zh word pair
    ja_rate = 0
    zh_rate = 0
    cnt = 0
    for word1, word2 in wordPairList:
        cnt += 1
        idx1, dist1 = wordVecDict.distance(word1, word2)
        ja_rate += 1 - idx1 * 1.0 / ja_wc
        idx2, dist2 = wordVecDict.distance(word2, word1)
        zh_rate += 1 - idx2 * 1.0 / zh_wc
        item = (word1, word2, idx1, dist1, idx2, dist2)
        result.append(item)
        print word1, word2, idx1, dist1, idx2, dist2
    ja_rate /= cnt
    zh_rate /= cnt
    print "Total", cnt, "pairs, ja_rate", ja_rate, "zh_rate", zh_rate


<file_sep>


vocab_file = "output/vocab.cj_001"


def getVocab():
    vocab = set()
    for line in open(vocab_file):
        if len(line) >= 3:
            vocab.add(line.split()[0])
    return vocab

def count(vocab, filename, lang):
    dic = {}
    for line in open(filename):
        cnt = 0
        for w in line.split():
            v = lang + w
            if v in vocab:
                cnt += 1
        dic[cnt] = dic.get(cnt, 0) + 1
    return dic

if __name__ == "__main__":
    vocab = getVocab()
    zh = count(vocab, 'data/train.zh_parsed', '1')
    ja = count(vocab, 'data/train.ja_parsed', '2')
    zhLine = ""
    jaLine = ""
    for i in xrange(331):
        zhLine += "%i," % zh.get(i, 0)
        jaLine += "%i," % ja.get(i, 0)
    print zhLine[:-1]
    print jaLine[:-1]
<file_sep>

class AnalogyData():
    def __init__(self):
        self.read_analogy()
        self.translationPairs = None
        self.crossZh = None
        self.crossJa = None
        self.cross2Zh2Ja = None
        self.cross1Zh3Ja = None
        self.cross3Zh1Ja = None

    def read_analogy(self):
        self.records = []
        fin = open("data/analogy_02/analogy_source.txt", 'r')
        for i in xrange(7):
            line = fin.readline()
            words = line.split()
            # in cjboc data file, 0 - ja, 1 - zh
            # in cjglo data file, 1 - zh, 2 - ja
            zh_county = '1' + words[0]
            zh_city = '1' + words[1]
            ja_county = '2' + words[2]
            ja_city = '2' + words[3]
            record = [[zh_county, zh_city], [ja_county, ja_city]]
            self.records.append(record)

    # in the translation pairs
    def generateTranslationPairs(self):
        self.translationPairs = []
        for record in self.records:
            # self.translationPairs.append((token.zh_county, token.zh_city, token.ja_county, token.ja_city))
            # self.translationPairs.append((token.zh_city, token.zh_county, token.ja_city, token.ja_county))
            # self.translationPairs.append((token.ja_county, token.ja_city, token.zh_county, token.zh_city))
            # self.translationPairs.append((token.ja_city, token.ja_county, token.zh_city, token.zh_county))
            for lang in [0, 1]:
                for c in [0, 1]:
                    self.translationPairs.append(record[lang][c], record[lang][1-c], record[1-lang][c], record[1-lang][1-c])

    def getTranslationPairs(self):
        if not self.translationPairs:
            self.generateTranslationPairs() 



    def generateCrossSingleLang(self, lang):
        result = []
        for record1 in self.records:
            for record2 in self.records:
                if record1 == record2:
                    continue
                for c in [0, 1]:
                    result.append((record1[lang][c], record1[lang][1-c], record2[lang][c], record2[lang][1-c]))
        return result



    def generateCrossZh(self):
        self.crossZh = self.generateCrossSingleLang(0)

    def getCrossZh(self):
        if not self.crossZh:
            self.generateCrossZh()
        return self.crossZh


        
    def generateCrossJa(self):
        self.crossJa = self.generateCrossSingleLang(1)

    def getCrossJa(self):
        if not self.crossJa:
            self.generateCrossJa()
        return self.crossJa



    def generateCross2Zh2Ja(self):
        self.cross2Zh2Ja = []
        for record1 in self.records:
            for record2 in self.records:
                if record1 == record2:
                    continue
                for c in [0, 1]:
                    self.cross2Zh2Ja.append((record1[0][c], record1[0][1-c], record2[1][c], record2[1][1-c]))
                    self.cross2Zh2Ja.append((record1[0][c], record1[1][1-c], record2[0][c], record2[1][1-c]))
                    self.cross2Zh2Ja.append((record1[0][c], record1[1][1-c], record2[1][c], record2[0][1-c]))
                    self.cross2Zh2Ja.append((record1[1][c], record1[0][1-c], record2[0][c], record2[1][1-c]))
                    self.cross2Zh2Ja.append((record1[1][c], record1[0][1-c], record2[1][c], record2[0][1-c]))
                    self.cross2Zh2Ja.append((record1[1][c], record1[1][1-c], record2[0][c], record2[0][1-c]))

    def getCross2Zh2Ja(self):
        if not self.cross2Zh2Ja:
            self.generateCross2Zh2Ja()
        return self.cross2Zh2Ja


    def generateCross1vs3(self, lang):
        # lang has 1 token
        result = []
        for record1 in self.records:
            for record2 in self.records:
                if record1 == record2:
                    continue
                for c in [0, 1]:
                    result.append((record1[lang][c], record1[1-lang][1-c], record2[1-lang][c], record2[1-lang][1-c]))
                    result.append((record1[1-lang][c], record1[lang][1-c], record2[1-lang][c], record2[1-lang][1-c]))
                    result.append((record1[1-lang][c], record1[1-lang][1-c], record2[lang][c], record2[1-lang][1-c]))
                    result.append((record1[1-lang][c], record1[1-lang][1-c], record2[1-lang][c], record2[lang][1-c]))
        return result



    def generateCross1Zh3Ja(self):
        self.cross1Zh3Ja = self.generateCross1vs3(0)

    def getCross1Zh3Ja(self):
        if not self.cross1Zh3Ja:
            self.generateCross1Zh3Ja()
        return self.cross1Zh3Ja



    def generateCross3Zh1Ja(self):
        self.cross3Zh1Ja = self.generateCross1vs3(1)

    def getCross3Zh1Ja(self):
        if not self.cross3Zh1Ja:
            self.generateCross3Zh1Ja()
        return self.cross3Zh1Ja



    def getByZhCount(self, zhCount):
        if zhCount == 0:
            return self.getCrossJa()
        elif zhCount == 1:
            return self.getCross1Zh3Ja()
        elif zhCount == 2:
            return self.getCross2Zh2Ja()
        elif zhCount == 3:
            return self.getCross3Zh1Ja()
        else:
            return self.getCrossZh()




if __name__ == '__main__':
    data = AnalogyData()
    #for t in data.getCross3Zh1Ja():
    #for t in data.getCross1Zh3Ja():
    #for t in data.getCross2Zh2Ja():
    for t in data.getCrossZh():
        print t[0], t[1], t[2], t[3]


<file_sep>

from WordVecDict import WordVecDict
import numpy as np


def readTestDataFile(filename, prefix):
    data = []
    for line in open(filename, 'r').readlines():
        if len(line) == 0:
            continue
        sentence = []
        for w in line.split():
            sentence.append(prefix + w)
        data.append(sentence)
    return data

def readTestData():
    zhData = readTestDataFile("data/test.zh_parsed", "1")
    jaData = readTestDataFile("data/test.ja_parsed", "2")
    return zhData, jaData

def calculateAcc(vecFile):
    wordVecDict = WordVecDict("output/vocab.cj_001", vecFile)
    zhData, jaData = readTestData()
    n = len(zhData)
    zhVec = []
    jaVec = []
    for i in xrange(n):
        zhVec.append(wordVecDict.sentenceVec(zhData[i]))
        jaVec.append(wordVecDict.sentenceVec(jaData[i]))
    
    matrix = []
    for i in xrange(n):
        line = []
        for j in xrange(n):
            line.append(np.dot(zhVec[i], jaVec[j]))
        matrix.append(line)


    dic = {}
    acc = 0
    for i in xrange(n):
        order = [(matrix[i][j], j) for j in xrange(n)]
        order.sort()
        for j in xrange(n):
            if order[j][1] == i:
                #print "Zh", i, "matches Ja", i, "at position", n - j
                k = n - j # oerder is from small to big, so k == 1 is the best match
                dic[k] = dic.get(k, 0) + 1
                acc += 1.0 - (k - 1.0) / n 
                break

    acc = acc * 1.0 / n
    return acc


def batch():
    for s in xrange(1, 6):
        for c in xrange(1, 6):
            inputFile = "output/batch_02/exp_s_0.0%i_c_0.0%i.vec.txt" % (s, c)
            print calculateAcc(inputFile), 
        print ""


if __name__ == "__main__":
    #print calculateAcc("output/batch_02/ctrl_s_0.03.vec.txt")
    batch()




<file_sep>
#include <stdio.h>
#include "bisearch_mapping_table.h"


extern MAPTABREC **k2sc, **sc2k;


void test_mapping_table_1(index) {
    int result;

    result = bisearch_table(sc2k, sc2k[index]->original, 0, SC2K_SIZE - 1);
    fprintf(stderr, "result = %d\n", result);

    result = bisearch_table(k2sc, k2sc[index]->original, 0, K2SC_SIZE - 1);
    fprintf(stderr, "result = %d\n", result);
}

void test_mapping_table_2() {
    int result;

    result = bisearch_table(sc2k, sc2k[SC2K_SIZE - 1]->original, 0, SC2K_SIZE - 1);
    fprintf(stderr, "result = %d (%d)\n", result, SC2K_SIZE - 1);

    result = bisearch_table(k2sc, k2sc[K2SC_SIZE - 1]->original, 0, K2SC_SIZE - 1);
    fprintf(stderr, "result = %d (%d)\n", result, K2SC_SIZE - 1);
}

void test_search() {
    test_mapping_table_1(0);
    test_mapping_table_1(7);
    test_mapping_table_1(77);
    test_mapping_table_1(777);
    test_mapping_table_1(5000);
    test_mapping_table_2();
}


void test_mapping() {

    int result;

    result = has_common_character("樱花", "桜がさく", 1);
    fprintf(stderr, "樱花 - 桜がさく, result = %d\n", result);

    result = has_common_character("桜がさく", "樱花", 2);
    fprintf(stderr, "桜がさく - 樱花, result = %d\n", result);

    result = has_common_character("国家", "桜がさく", 1);
    fprintf(stderr, "国家 - 桜がさく, result = %d\n", result);

    result = has_common_character("国家", "王国", 2);
    fprintf(stderr, "国家 - 王国, result = %d\n", result);
}

int main(int argc, char **argv) {
    load_mapping_tables();
    test_mapping();
}<file_sep>/* Author <NAME>, April 2001.
   Permission to use this code is freely granted, provided that this
   statement is retained. */

/* Bitwise hash function.  Note that tsize does not have to be prime. */
unsigned int
bitwisehash(char *word, int tsize, unsigned int seed)
{
    char	c;
    unsigned int h;

    h = seed;
    for( ; ( c=*word )!='\0' ; word++ )
    {
	h ^= ( (h << 5) + c + (h >> 2) );
    }
    return((unsigned int)((h&0x7fffffff) % tsize));
}

/* Efficient string comparison */
int scmp( char *s1, char *s2 ) {
    while (*s1 != '\0' && *s1 == *s2) {s1++; s2++;}
    return(*s1 - *s2);
}<file_sep>from collections import defaultdict

"""
    The input file is download from:
      
    http://lotus.kuee.kyoto-u.ac.jp/~chu/pubdb/LREC2012/kanji_mapping_table.txt

    Removed the introduction text in the head, only character triples remained
    in input file.
"""


def check_kanji_length():
    """
        Checked by this function, the first item of each triple is excatly one
        Kanji charachter.
    """
    fin = open('data/kanji_mapping_table_clean.txt', 'r')
    for line in fin:
        triple = line.split()
        kanji = triple[0]
        if len(kanji) != 3:
            print kanji, len(kanji)
            break

def check_duplicated_kanji():
    """
        Checked by this function, No duplicated Kanjis in this mapping table's
        first column
    """
    fin = open('data/kanji_mapping_table_clean.txt', 'r')
    kanji_set = set()
    for line in fin:
        triple = line.split()
        kanji = triple[0]
        if kanji in kanji_set:
            print kanji
            break
        kanji_set.add(kanji)
    print "There are", len(kanji_set), "Kanjis in the mapping table."

def write_tuple_to_file(tuple, filename):
    fout = open(filename, 'w')
    for kanji, simple_chinese in tuple:
        fout.write("%s %s\n" % (kanji, simple_chinese))
    fout.close()

def kanji_2_simple_chinese():
    """
        Result:
        There are 569 N/A, 5205 single corresponding Simple Chinese 
        charachters, 582 mutiple corresponding Simple Chinese charachters,
        in total 6356 lines.
        A Kanji Character is mapping to X Simple Chinese Character:
        {0: 569, 1: 5205, 2: 538, 3: 41, 4: 1, 6: 2}
    """
    fin = open('data/kanji_mapping_table_clean.txt', 'r')
    single_result = []
    mutiple_result = []
    single = 0
    mutiple = 0
    na = 0
    dic = {}
    for line in fin:
        triple = line.split()
        kanji = triple[0]
        simple_chinese = triple[2]
        if simple_chinese == 'N/A':
            na += 1
        elif len(simple_chinese) <= 3:
            single += 1
            single_result.append((kanji, simple_chinese))
        else:
            mutiple += 1
            simple_chinese = ''.join(simple_chinese.split(','))
            mutiple_result.append((kanji, simple_chinese))
            key = len(simple_chinese) / 3
            dic[key] = dic.get(key, 0) + 1

    # write_tuple_to_file(single_result, "output/mapping_table/single_01.txt")
    # write_tuple_to_file(mutiple_result, "output/mapping_table/mutiple_01.txt")

    result = single_result + mutiple_result
    result.sort()
    write_tuple_to_file(result, "output/mapping_table/kanji2simplec.txt")

    print "There are", na, "N/A,", 
    print single, "single corresponding Simple Chinese charachters,",
    print mutiple, "mutiple corresponding Simple Chinese charachters,",
    print "in total", (na + single + mutiple), "lines."

    dic[0] = na
    dic[1] = single
    print "A Kanji Character is mapping to X Simple Chinese Character:"
    print dic

def simple_chinese_2_kanji():
    """
        A Simple Chinese Character is mapping to X Kanji Character:
        {1: 3873, 2: 903, 3: 189, 4: 34, 5: 5, 6: 1, 7: 1}
    """
    fin = open('data/kanji_mapping_table_clean.txt', 'r')
    dic = defaultdict(list)
    for line in fin:
        triple = line.split()
        kanji = triple[0]
        simple_chinese = triple[2]
        if simple_chinese == 'N/A':
            continue
        elif len(simple_chinese) <= 3:
            dic[simple_chinese].append(kanji)
        else:
            for character in simple_chinese.split(','):
                dic[character].append(kanji)

    result = [(x, ''.join(y)) for x, y in dic.items()]
    result.sort()
    write_tuple_to_file(result, "output/mapping_table/simplec2kanji.txt")
    
    dic2 = defaultdict(int)
    for key in dic:
        dic2[len(dic[key])] += 1

    print "A Simple Chinese Character is mapping to X Kanji Character:"
    print dic2

def find_same_shape_character():
    """
        The shape of a common character in 3 categories are sometimes the same, sometimes not.
        Only consider lines without "N/A", there are 5 patterns:
        Type   Kanji           Tranditional    Simple
        0      A               A               A
        1      A               A               B
        2      A               B               A
        3      A               B               B
        4      A               B               C

        This function counts these types, and save to corresponding files.
    """
    fin = open('data/kanji_mapping_table_clean.txt', 'r')
    dic = defaultdict(list)

    type_set = [[] for _ in xrange(6)] # Type 5 is 'N/A'

    for line in fin:
        triple = line.split()
        if 'N/A' in triple:
            type_set[5].append(line)
            continue
        kanji = triple[0]
        tranditional = triple[1]
        simple = triple[2]
        for c_kanji in kanji.split(','):
            for c_tranditional in tranditional.split(','):
                for c_simple in simple.split(','):
                    if c_kanji == c_tranditional and c_kanji == c_simple:
                        type_set[0].append(line)
                    elif c_kanji == c_tranditional:
                        type_set[1].append(line)
                    elif c_kanji == c_simple:
                        type_set[2].append(line)
                    elif c_tranditional == c_simple:
                        type_set[3].append(line)
                    else:
                        type_set[4].append(line)

    for i in xrange(6):
        fout = open("output/mapping_table/character_shape_type_%i.txt" % i, "w")
        for line in type_set[i]:
            fout.write(line)
        fout.close()

def find_same_shape_character_single():
    """
        Same with find_same_shape_character(), but only deal with line that in each type
        there is only 1 character. 
    """
    fin = open('data/kanji_mapping_table_clean.txt', 'r')
    dic = defaultdict(list)

    type_set = [[] for _ in xrange(6)] # Type 5 is line with ','

    for line in fin:
        if ',' in line:
            type_set[5].append(line)
            continue
        triple = line.split()
        if 'N/A' in triple:
            continue
        c_kanji = triple[0]
        c_tranditional = triple[1]
        c_simple = triple[2]
        if c_kanji == c_tranditional and c_kanji == c_simple:
            type_set[0].append(line)
        elif c_kanji == c_tranditional:
            type_set[1].append(line)
        elif c_kanji == c_simple:
            type_set[2].append(line)
        elif c_tranditional == c_simple:
            type_set[3].append(line)
        else:
            type_set[4].append(line)

    for i in xrange(6):
        fout = open("output/mapping_table/character_shape_type_single_%i.txt" % i, "w")
        for line in type_set[i]:
            fout.write(line)
        fout.close()

if __name__ == '__main__':
    # check_kanji_length()
    # check_duplicated_kanji()
    # kanji_2_simple_chinese()
    # simple_chinese_2_kanji()
    # find_same_shape_character()
    find_same_shape_character_single()

<file_sep>//  Tool to calculate word-word cooccurrence statistics for sentence aligned
//  corpus of 2 languages.
//
//  Copyright (c) 2014 The Board of Trustees of
//  The Leland Stanford Junior University. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
//
//
//  For more information, bug reports, fixes, contact:
//    <NAME> (<EMAIL>)


#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>


typedef struct mapping_table {
  //char *original;
  char original[4];
  // at most 7 corresponding characters
  //char *corresponding;//
  char corresponding[22];
} mapping_table;

mapping_table **k2sc, **sc2k;


char *sc2k_filename = "../data/simplec2kanji.txt";
char *k2sc_filename = "../data/kanji2simplec.txt";

int sc2k_size = 5;//5006;
int k2sc_size = 5787;

int debug_counter = 0;



int compare_original(const void *a, const void *b) {
    //int t = strcmp(((mapping_table *)a)->original, ((mapping_table *)b)->original);
    //fprintf(stderr, "%s %s %d\n", ((mapping_table *)a)->original, ((mapping_table *)b)->original, t);
    //scanf("%d", &t);
    mapping_table *aa, *bb, **aaa, **bbb;
    aaa = ((mapping_table **)a);
    //aa = aaa[0];
    aa = *aaa;
    bbb = ((mapping_table **)b);
    bb = *bbb;
    //bb = bbb[0];
    fprintf(stderr, "Compareing: %s %s\n", (*(mapping_table **)a)->original, (*(mapping_table **)b)->original);
    fprintf(stderr, "    &a=%p, &b=%p, \n  &aa=%p, &bb=%p, \n  &aa->original=%p, bb->original=%p\n", a, b, aa, bb, aa->original, bb->original);
    //return strcmp(((mapping_table *)a)->original, ((mapping_table *)b)->original);
    return strcmp((*(mapping_table **)a)->original, (*(mapping_table **)b)->original);
}

int compare_original_2(mapping_table *a, mapping_table *b) {
    int t;
    if (debug_counter < 70000) {
        t = strcmp(a->original, b->original);
//        fprintf(stderr, "%s %s %x %x %d\n", a->original, b->original, a, b, t);
        scanf("%d", &t);
        debug_counter++;
    }
    return strcmp(a->original, b->original);
}

int get_characters(char *word, FILE *fin) {
    int i = 0, ch;
    while (!feof(fin)) {
        ch = fgetc(fin);
        //if (ch == 13) continue;
        if ((ch == ' ') || (ch == '\t') || (ch == '\n')) {
            break;
        }
        word[i++] = ch;
        //if (i >= MAX_STRING_LENGTH - 1) i--;   // truncate words that exceed max length
    }
    word[i] = 0;
    return i;
}


mapping_table ** ReadMappingTableFromFile(char *filename, int table_size) {
    int i;
    FILE *f;
    char *original, *corresponding;
    mapping_table *table_record;
    mapping_table **table;
    table = (mapping_table **)malloc( sizeof(mapping_table *) * table_size );

    f = fopen(filename, "rb");
    for (i = 0; i < table_size; i++) {
// for char * original.
        // table_record = (mapping_table *)malloc( sizeof(mapping_table));
        // table_record->original = (char *)malloc( sizeof(char) * 30);
        // if (i % 700 == 0) fprintf(stderr, "table_record->original %p\n", table_record->original);
        // table_record->corresponding = (char *)malloc( sizeof(char) * 50);
        // get_characters(table_record->original, f);
        // get_characters(table_record->corresponding, f);
        // //table_record->original = original;
        // //table_record->corresponding = corresponding;
        // table[i] = table_record;
        // if (i % 700 == 0) fprintf(stderr, "%d %s %s %p %p %p\n", i, table[i]->original, table[i]->corresponding, table[i]->original, table[i]->corresponding, table[i]);
// for char original[4]         
        table_record = (mapping_table *)malloc( sizeof(mapping_table) );
        get_characters(table_record->original, f);
        get_characters(table_record->corresponding, f);
        //table_record->original = original;
        //table_record->corresponding = corresponding;
        table[i] = table_record;
        //if (i % 700 == 0) fprintf(stderr, "%d %s %s %p %p %p\n", i, table[i]->original, table[i]->corresponding, table[i]->original, table[i]->corresponding, table[i]);

    }
    fclose(f);

    //fprintf(stderr, "ReadMappingTableFromFile - table %p, table[7] %p, table[7]->original %p\n", table, table[7], table[7]->original);
    fprintf(stderr, "=============== After Reading File ============\n");
    fprintf(stderr, "&table=%p\n", table);
    for (i = 0; i < sc2k_size; i++) {
        fprintf(stderr, "%d ", i);
        fprintf(stderr, "table[i]=%s, &table[i]=%p, &table[i]->original=%p\n", table[i]->original, table[i], table[i]->original);
    }

    // TODO the bug is because the usage of qsort!!!!!!!===============
    qsort(&table[0], table_size, sizeof(mapping_table*), compare_original);

    fprintf(stderr, "=============== After QSORT ============\n");
    fprintf(stderr, "&table=%p\n", table);
    for (i = 0; i < sc2k_size; i++) {
        fprintf(stderr, "%d ", i);
        fprintf(stderr, "table[i]=%s, &table[i]=%p, &table[i]->original=%p\n", table[i]->original, table[i], table[i]->original);
    }

    // fprintf(stderr, "ReadMappingTableFromFile - table %p, table[7] %p, table[7]->original %p\n", table, table[7], table[7]->original);

    // for (i = 0; i < sc2k_size; i += 700) {
    //     fprintf(stderr, "%d ", i);
    //     fprintf(stderr, "%p %p\n", table[i]->original, table[i]->corresponding);//, table[i]->original, table[i]->corresponding);
    // }

    return table;
}



void ReadMappingTable() {
    sc2k = ReadMappingTableFromFile(sc2k_filename, sc2k_size);
    k2sc = ReadMappingTableFromFile(k2sc_filename, k2sc_size);
}


int BinarySearchTable(mapping_table **table, char *ch, int left, int right) {
    int mid, cmp;
    if (right < left) {
        return -1;
    }
    mid = left + (right - left) / 2;
    fprintf(stderr, "mid = %d, %s\n", mid, table[mid]->original);
    
    //cmp = strcmp(ch, table[mid]->original);
    
    cmp = strcmp(table[mid]->original, ch);


//int compare_original(const void *a, const void *b) {
//    return strcmp(((struct mapping_table *)a)->original, ((struct mapping_table *)b)->original);
//}
    if (cmp == 0) {
        return mid;
    }
    else if (cmp < 0) {
        return BinarySearchTable(table, ch, left, mid - 1);
    }
    else {
        return BinarySearchTable(table, ch, mid + 1, right);
    }
}



int test_mapping_table_1() {
    int i, result;
    fprintf(stderr, "test_mapping_table_1\n");
    sc2k = ReadMappingTableFromFile(sc2k_filename, sc2k_size);
    fprintf(stderr, "After ReadMappingTableFromFile\n");
    //fprintf(stderr, "%p\n", sc2k);
    fprintf(stderr, "table %p, table[7] %p, table[7]->original %p\n", sc2k, sc2k[7], sc2k[7]->original);
    for (i = 0; i < sc2k_size; i++) {
        //if (i % 700 == 0) fprintf(stderr, "%d ", i);
        //if (i % 700 == 0) fprintf(stderr, "%p %p\n", sc2k[i]->original, sc2k[i]->corresponding);
        printf("%s %s %p %p %p\n", sc2k[i]->original, sc2k[i]->corresponding, sc2k[i]->original, sc2k[i]->corresponding, sc2k[i]);
    }
    result = BinarySearchTable(sc2k, sc2k[1]->original, 0, sc2k_size - 1);
    fprintf(stderr, "result = %d\n", result);

}



int main(int argc, char **argv) {
    test_mapping_table_1();
}

<file_sep>//  Tool to calculate word-word cooccurrence statistics for sentence aligned
//  corpus of 2 languages.
//
//  Copyright (c) 2014 The Board of Trustees of
//  The Leland Stanford Junior University. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
//
//
//  For more information, bug reports, fixes, contact:
//    <NAME> (<EMAIL>)


#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include "hash_mapping_table.h"

MAPTABREC **k2sc, **sc2k;

/* Create hash table, initialise ptrs to NULL */
MAPTABREC ** init_hash_mapping_table() {
    int i;
    MAPTABREC **ht;
    ht = (MAPTABREC **) malloc( sizeof(MAPTABREC *) * M_SIZE );
    for(i = 0; i < M_SIZE; i++) {
        ht[i] = (MAPTABREC *) NULL;
    }
    return(ht);
}

/* Search hash table for given string, return record if found, else NULL */
MAPTABREC * hash_search_mapping_table(MAPTABREC **ht, char *w) {
    MAPTABREC *htmp, *hprv;
    unsigned int hval = M_HASHFN(w, M_SIZE, M_SEED);

    for( hprv = NULL, htmp=ht[hval]
        ; htmp != NULL && scmp(htmp->original, w) != 0
        ; hprv = htmp, htmp = htmp->next )
    {
    ;
    }

    if( hprv!=NULL ) {
        /* move to front on access */
        hprv->next = htmp->next;
        htmp->next = ht[hval];
        ht[hval] = htmp;
    }

    return(htmp);
}

/* Search hash table for given string, insert if not found */
void hash_insert_mapping_table(MAPTABREC **ht, char *original, char *corresponding) {
    MAPTABREC *htmp, *hprv;
    unsigned int hval = M_HASHFN(original, M_SIZE, M_SEED);

    for( hprv = NULL, htmp=ht[hval]
        ; htmp != NULL && scmp(htmp->original, original) != 0
        ; hprv = htmp, htmp = htmp->next ) 
    {
    ;
    }

    if(htmp == NULL) {
        htmp = (MAPTABREC *) malloc( sizeof(MAPTABREC) );
        //htmp->word = (char *) malloc( strlen(w) + 1 );
        strcpy(htmp->original, original);
        strcpy(htmp->corresponding, corresponding);
        htmp->next = NULL;
        if(hprv == NULL) {
            ht[hval] = htmp;
        }
        else {
            hprv->next = htmp;
        }
        /* new records are not moved to front */
    }
    else {
        if(hprv != NULL) {
            /* move to front on access */
            hprv->next = htmp->next;
            htmp->next = ht[hval];
            ht[hval] = htmp;
        }
    }
    return;
}

/**
 * Read one or several Kanji or Hanzi from text file
 */
int get_characters(char *word, FILE *fin) {
    int i = 0, ch;
    while (!feof(fin)) {
        ch = fgetc(fin);
        if ((ch == ' ') || (ch == '\t') || (ch == '\n')) {
            break;
        }
        word[i++] = ch;
    }
    word[i] = 0;
    return i;
}

/**
 * Read mapping table from file, and sort it
 */
MAPTABREC ** load_one_mapping_table(char *filename, int table_size) {
    int i;
    FILE *f;
    char original[4], corresponding[22];
    // MAPTABREC *record;
    MAPTABREC **table;
    table = init_hash_mapping_table();
    fprintf(stderr, "start reading %s\n", filename);
    f = fopen(filename, "rb");
    for (i = 0; i < table_size; i++) {      
        // record = (MAPTABREC *)malloc( sizeof(MAPTABREC) );
        get_characters(original, f);
        get_characters(corresponding, f);
        hash_insert_mapping_table(table, original, corresponding);
    }
    fclose(f);


    return table;
}

/**
 * Read 2 mapping tables
 */
void load_mapping_tables() {
    sc2k = load_one_mapping_table(SC2K_FILENAME, SC2K_SIZE);
    k2sc = load_one_mapping_table(K2SC_FILENAME, K2SC_SIZE);
}


int compare_characters(char *corresponding, char *word2) {
  int len1, len2, i, j;
  fprintf(stderr, "compare_characters: %s %s\n", corresponding, word2);
  len1 = strlen(corresponding);
  len2 = strlen(word2);
  fprintf(stderr, "compare_characters: %d %d\n", len1, len2);
  for (i = 0; i < len1; i+= 3) {
    for (j = 0; j < len2; j += 3) {
      if (word2[j] == corresponding[i] && word2[j+1] == corresponding[i+1] && word2[j+2] == corresponding[i+2]) {
        //if (verbose > 2) fprintf(stderr, "source_ch %s, target_ch %s, j %d, word2 %s\n", source_ch, target_ch, j, word2);
        return 1;
      }
    }
  }
  return 0;
}

/**
 * return 1 if the 2 words has a common character.
 */
int has_common_character(char *word1, char *word2, int lang_id) {
  char source_ch[4], target_ch[4];
  int len1, len2;
  int i, j, table_size;
  int parallel = 1 - lang_id;
  MAPTABREC **table;
  MAPTABREC *ret;

  if (lang_id == 1) {
    table = sc2k;
    table_size = SC2K_SIZE;
  }
  else {
    table = k2sc;
    table_size = K2SC_SIZE;
  }

  len1 = strlen(word1);
  len2 = strlen(word2);
  if (len1 % 3 != 0 || len2 % 3 != 0) {
    return 0;
  }
  fprintf(stderr, "has_common_character : %s - %s - %d\n", word1, word2, lang_id);
  source_ch[3] = 0;
  target_ch[3] = 0;
  for (i = 0; i < len1; i += 3) {
    // get a character from word1
    for (j = 0; j < 3; j++) {
      source_ch[j] = word1[i + j];
    }
    // pos = bisearch_table(table, source_ch, 0, table_size - 1);
    // if (pos >= 0) {
    //     if (compare_characters(table[pos]->corresponding, word2) > 0) {
    //         return 1;
    //     }
    // }
    fprintf(stderr, "for : %s\n", source_ch);
    ret = hash_search_mapping_table(table, source_ch);
    fprintf(stderr, "for : %p %s %s\n", ret, ret->original, ret->corresponding);
    if (ret) {
        if (compare_characters(ret->corresponding, word2) > 0) {
            return 1;
        }
    }
  }
  return 0;
}



<file_sep>CC = gcc
#For older gcc, use -O3 or -O2 instead of -Ofast
CFLAGS = -lm -pthread -Ofast -march=native -funroll-loops -Wno-unused-result
BUILDDIR := build
SRCDIR := src

all: dir glove shuffle cooccur vocab_count bi_vocab_count bi_cooccur

dir :
	mkdir -p $(BUILDDIR)
glove : $(SRCDIR)/glove.c
	$(CC) $(SRCDIR)/glove.c -o $(BUILDDIR)/glove $(CFLAGS)
shuffle : $(SRCDIR)/shuffle.c
	$(CC) $(SRCDIR)/shuffle.c -o $(BUILDDIR)/shuffle $(CFLAGS)
cooccur : $(SRCDIR)/cooccur.c
	$(CC) $(SRCDIR)/cooccur.c -o $(BUILDDIR)/cooccur $(CFLAGS)
vocab_count : $(SRCDIR)/vocab_count.c
	$(CC) $(SRCDIR)/vocab_count.c -o $(BUILDDIR)/vocab_count $(CFLAGS)
bi_vocab_count : $(SRCDIR)/bi_vocab_count.c
	$(CC) $(SRCDIR)/bi_vocab_count.c -o $(BUILDDIR)/bi_vocab_count $(CFLAGS)
bi_cooccur : $(SRCDIR)/bi_cooccur.c
	#$(CC) $(SRCDIR)/bi_cooccur.c $(SRCDIR)/hash_mapping_table.c -o $(BUILDDIR)/bi_cooccur $(CFLAGS)
	$(CC) $(SRCDIR)/bi_cooccur.c $(SRCDIR)/bisearch_mapping_table.c -o $(BUILDDIR)/bi_cooccur $(CFLAGS)

clean:
	rm -rf glove shuffle cooccur vocab_count bi_vocab_count bi_cooccur build
<file_sep># CJ-Glo
Chinese Japanese Word Embedding based on GloVe
<file_sep>

from WordVecDict import WordVecDict
from AnalogyData import AnalogyData

data = AnalogyData()
ja_wc = 22748
zh_wc = 25219

def calculateRate(idx, word):
    if word[0] == '1':
        return 1 - idx * 1.0 / zh_wc
    else:
        return 1 - idx * 1.0 / ja_wc

def batchCtrl():
    for b in xrange(1, 6):
        inputFile = "output/batch_02/ctrl_s_0.0%i.vec.txt" % b
        wordVecDict = WordVecDict("output/vocab.cj_001", inputFile)
        for zhCount in xrange(5):
            outputFile = "output/batch_02/analogy_01/ctrl_s_0.0%i_zh_%i.csv" % (b, zhCount)
            print "Writing to", outputFile, '... ... ... ...'
            fout = open(outputFile, 'w')
            records = data.getByZhCount(zhCount)
            for record in records:
                idx, _ = wordVecDict.analogy(record)
                rate = calculateRate(idx, record[3])
                line = ','.join(record)
                line += ',%i,%f\n' % (idx, rate)
                fout.write(line)
            fout.close()


def batchExp():
    for s in xrange(1, 6):
        for c in xrange(1, 6):
            inputFile = "output/batch_02/exp_s_0.0%i_c_0.0%i.vec.txt" % (s, c)
            wordVecDict = WordVecDict("output/vocab.cj_001", inputFile)
            for zhCount in xrange(5):
                outputFile = "output/batch_02/analogy_01/exp_s_0.0%i_c_0.0%i_zh_%i.csv" % (s, c, zhCount)
                print "Writing to", outputFile, '... ... ... ...'
                fout = open(outputFile, 'w')
                records = data.getByZhCount(zhCount)
                for record in records:
                    idx, _ = wordVecDict.analogy(record)
                    rate = calculateRate(idx, record[3])
                    line = ','.join(record)
                    line += ',%i,%f\n' % (idx, rate)
                    fout.write(line)
                fout.close()

if __name__ == '__main__':
    #batchCtrl()
    batchExp()<file_sep>

import WordVecDict

if __name__ == '__main__':
    #wordVecDict = WordVecDict.WordVecDict("output/vocab.cj_001", "output/batch_02/ctrl_s_0.03.vec.txt")
    wordVecDict = WordVecDict.WordVecDict("output/vocab.cj_001", "output/batch_02/exp_s_0.05_c_0.05.vec.txt")
    while True:
        print "Input w0 w1 w2 w3:"
        line = raw_input().split()
        print wordVecDict.distance(line[0], line[1], True)<file_sep>//  Tool to calculate word-word cooccurrence statistics for sentence aligned
//  corpus of 2 languages.
//
//  Copyright (c) 2014 The Board of Trustees of
//  The Leland Stanford Junior University. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
//
//
//  For more information, bug reports, fixes, contact:
//    <NAME> (<EMAIL>)


#define SC2K_FILENAME "../data/simplec2kanji.txt"
#define K2SC_FILENAME "../data/kanji2simplec.txt"

#define SC2K_SIZE 5006
#define K2SC_SIZE 5787

typedef struct mapping_table_record {
  char original[4];
  // at most 7 corresponding characters
  char corresponding[22];
} MAPTABREC;

/**
 * Read 2 mapping tables
 */
void load_mapping_tables();

/**
 * Binary Search a character from the mapping table
 */
int bisearch_table(MAPTABREC **table, char *ch, int left, int right);

/**
 * return 1 if the 2 words has a common character.
 */
int has_common_character(char *word1, char *word2, int lang_id);

<file_sep>


class Token():
    def __init__(self, zh_county, zh_city, ja_county, ja_city):
        self.zh_county = zh_county
        self.zh_city = zh_city
        self.ja_county = ja_county
        self.ja_city = ja_city



def read_analogy():
    fin = open("data/analogy_02/analogy_source.txt", 'r')
    for i in xrange(7):
        fin.readline()
        line = fin.readline()
        tokens = line.split()
                # in data file, 0 - ja, 1 - zh
                for k in xrange(4):
                    if tokens[k] == 0:







if __name__ == "__main__":
<file_sep>"""
    Experiment 2
    Use a file of word pairs, to calculate Cross-Lingual Synonym Comparison 
"""

import argparse
import numpy as np
import sys

def generate():
    parser = argparse.ArgumentParser()
    parser.add_argument('--vocab_file', default='vocab.txt', type=str)
    parser.add_argument('--vectors_file', default='vectors.txt', type=str)
    args = parser.parse_args()

    with open(args.vocab_file, 'r') as f:
        words = [x.rstrip().split(' ')[0] for x in f.readlines()]
    with open(args.vectors_file, 'r') as f:
        vectors = {}
        for line in f:
            vals = line.rstrip().split(' ')
            vectors[vals[0]] = [float(x) for x in vals[1:]]

    vocab_size = len(words)
    vocab = {w: idx for idx, w in enumerate(words)}
    ivocab = {idx: w for idx, w in enumerate(words)}

    vector_dim = len(vectors[ivocab[0]])
    W = np.zeros((vocab_size, vector_dim))
    for word, v in vectors.items():
        if word == '<unk>':
            continue
        W[vocab[word], :] = v

    # normalize each word vector to unit variance
    W_norm = np.zeros(W.shape)
    d = (np.sum(W ** 2, 1) ** (0.5))
    W_norm = (W.T / d).T
    return (W_norm, vocab, ivocab)


def distance(W, vocab, ivocab, word1, word2):
    if not word1 in vocab:
        print('Word: %s  Out of dictionary!\n' % word1)
        return
    if not word2 in vocab:
        print('Word: %s  Out of dictionary!\n' % word2)
        return
    vec_result = np.copy(W[vocab[word1], :])
    
    vec_norm = np.zeros(vec_result.shape)
    d = (np.sum(vec_result ** 2,) ** (0.5))
    vec_norm = (vec_result.T / d).T

    dist = np.dot(W, vec_norm.T)

    #for term in input_term.split(' '):
    index = vocab[word1]
    dist[index] = -np.Inf

    a = np.argsort(-dist)#[:N]

    #print("\n                               Word       Cosine distance\n")
    #print("---------------------------------------------------------\n")
    #for x in a:
    #    print("%35s\t\t%f" % (ivocab[x], dist[x]))

    idx = 0
    for x in a:
        if ivocab[x][0] == word2[0]:
            idx += 1
            if ivocab[x] == word2:
                #print("%s -> %s %f" % (word1, ivocab[x], dist[x]))
                #break
                return idx, dist[x]
    print("Error: %s -> %s %f" % (word1, word2))
    return -1, -1

def readWordPairs():
    path = "experiment/experiment2_data.txt"
    fin = open(path, 'r')
    wordPairList = []
    for line in fin.readlines():
        if len(line) > 2:
            words = line.split()
            w1 = "2" + words[0].strip()
            w2 = "1" + words[1].strip()
            wordPairList.append((w1, w2))
    return wordPairList

if __name__ == "__main__":
    N = 20;          # number of closest words that will be shown
    print "reading word embeddings"
    W, vocab, ivocab = generate()
    print "reading word pairs"
    wordPairList = readWordPairs()
    print "calculating distances"

    ja_wc = 22748
    zh_wc = 25219

    result = []
    # ja, zh word pair
    ja_rate = 0
    zh_rate = 0
    cnt = 0
    for word1, word2 in wordPairList:
        cnt += 1
        idx1, dist1 = distance(W, vocab, ivocab, word1, word2)
        ja_rate += 1 - idx1 * 1.0 / ja_wc
        idx2, dist2 = distance(W, vocab, ivocab, word2, word1)
        zh_rate += 1 - idx2 * 1.0 / zh_wc
        item = (word1, word2, idx1, dist1, idx2, dist2)
        result.append(item)
        print word1, word2, idx1, dist1, idx2, dist2
    ja_rate /= cnt
    zh_rate /= cnt
    print "Total", cnt, "pairs, ja_rate", ja_rate, "zh_rate", zh_rate


<file_sep>"""
    Train CJ-GLO model by different parameters, then test the result.
"""

import os

vocab_file = "output/vocab.cj_001"

glove_args_2 = {
    "threads": "8",
    "x-max": "10",
    "iter": "15",
    "vector-size": "50",
    "binary": "2",
    "vocab-file": vocab_file,
    "verbose": "0"
}

# used to find a good ctrl parameters
glove_args = {
    "threads": "16",
    "x-max": "100",
    "iter": "15",
    "vector-size": "100",
    "binary": "2",
    "vocab-file": vocab_file,
    "eta": "0.05",
    "verbose": "2",
    "alpha": "0.75"
}

def fill_glove_command(input_filename, output_filename):
    cmd = "build/glove -save-file output/" + output_filename
    cmd += " -input-file output/" + input_filename
    for args in glove_args:
        cmd += " -" + args + " " + glove_args[args]
    return cmd

shuffle_args_old = {
    "memory": "4.0",
    "verbose": "0"
}

shuffle_args = {
    "memory": "8.0",
    "verbose": "2"
}

def fill_shuffle_command(input_filename, output_filename):
    cmd = "build/shuffle"
    for args in shuffle_args:
        cmd += " -" + args + " " + shuffle_args[args]
    cmd += " < output/" + input_filename
    cmd += " > output/" + output_filename
    return cmd

def fill_bi_cooccur_command(window_size, sentence_rate, cjglo, cjglo_rate, output_filename):
    cmd = "build/bi_cooccur -verbose 0"
    cmd += " -corpus-file-1 data/train.zh_parsed"
    cmd += " -corpus-file-2 data/train.ja_parsed"
    cmd += " -window-size " + window_size
    cmd += " -vocab-file " + vocab_file
    cmd += " -sentence-rate " + sentence_rate
    cmd += " -cjglo " + cjglo
    cmd += " -cjglo-rate " + cjglo_rate
    cmd += " > output/" + output_filename
    return cmd

def fill_experiment2_command(word_vec_filename, output_filename):
    cmd = "python tools/experiment2.py --vocab_file " + vocab_file
    cmd += " --vectors_file output/" + word_vec_filename
    cmd += " > output/" + output_filename
    return cmd



def run_once(sentence_rate, cjglo, window_size, cjglo_rate, prefix):
    co_output_filename = prefix + ".co"
    co_cmd = fill_bi_cooccur_command(window_size, sentence_rate, cjglo, cjglo_rate, co_output_filename)
    print co_cmd
    os.popen(co_cmd)

    shuf_output_filename = prefix + ".shuf"
    shuf_cmd = fill_shuffle_command(co_output_filename, shuf_output_filename)
    print shuf_cmd
    os.popen(shuf_cmd)

    glove_output_filename = prefix + ".vec"
    glove_cmd = fill_glove_command(shuf_output_filename, glove_output_filename)
    print glove_cmd
    os.popen(glove_cmd)

    word_vec_filename = prefix + ".vec.txt"
    experiment2_output_filename = prefix + ".exp2.txt"
    experiment2_cmd = fill_experiment2_command(word_vec_filename, experiment2_output_filename)
    print experiment2_cmd
    os.popen(experiment2_cmd)

    # clean tmp files
    print "rm -rf output/" + co_output_filename
    os.popen("rm -rf output/" + co_output_filename)
    print "rm -rf output/" + shuf_output_filename
    os.popen("rm -rf output/" + shuf_output_filename)


def run_control_by_list(prefix, sentence_rate_list):
    for sentence_rate in sentence_rate_list:
        control_prefix = prefix + "ctrl_s_" + sentence_rate
        run_once(sentence_rate, "0", "0", control_prefix)


def run_experiment_by_list(prefix, window_size_list, sentence_rate_list, cjglo_rate_list):
    for sentence_rate in sentence_rate_list:
        for cjglo_rate in cjglo_rate_list:
            for window_size in window_size_list:
                exp_prefix = prefix + "exp_w_" + window_size + "_s_" + sentence_rate + "_c_" + cjglo_rate
                run_once(sentence_rate, "1", window_size, cjglo_rate, exp_prefix)
            

def main():
    prefix = "batch_04/"
    sentence_rate_list = ["0.00", "0.01", "0.02", "0.03", "0.04", "0.05"] # [str(0.1 * x) for x in xrange(1, 11)]
    #sentence_rate_list = ["0.000", "0.001", "0.002", "0.003", "0.004", "0.005", "0.006", "0.007", "0.008", "0.009"]
    cjglo_rate_list = ["0.00", "0.01", "0.02", "0.03", "0.04", "0.05"] # [str(0.1 * x) for x in xrange(1, 11)]
    #cjglo_rate_list = ["0.000", "0.001", "0.002", "0.003", "0.004", "0.005", "0.006", "0.007", "0.008", "0.009"]
    #run_control_by_list(prefix, sentence_rate_list)
    window_size_list = ["5", "10"]
    run_experiment_by_list(prefix, window_size_list, sentence_rate_list, cjglo_rate_list)


if __name__ == '__main__':
    main()<file_sep>

import WordVecDict

if __name__ == '__main__':
    #vecFile = "output/batch_02/exp_s_0.05_c_0.05.vec.txt"
    vecFile = "cjboc_output/batch_01/exp_s_0.05_c_0.05.cjboc.txt"
    #wordVecDict = WordVecDict.WordVecDict("output/vocab.cj_001", "output/batch_02/ctrl_s_0.03.vec.txt")
    wordVecDict = WordVecDict.WordVecDict("output/vocab.cj_001", )
    while True:
        print "Input w0 w1:"
        line = raw_input()
        print wordVecDict.analogy(line.split(), True)<file_sep>"""
    Count the sentence lenth of a parsed (after word segmentation) corpus.
"""
import argparse

def count_length():
    parser = argparse.ArgumentParser()
    parser.add_argument('--corpus_file', default='train.js_parsed', type=str)
    args = parser.parse_args()
    fin = open(args.corpus_file)
    dic = {}
    for line in fin:
        length = len(line.split())
        dic[length] = dic.get(length, 0) + 1
    return dic

if __name__ == '__main__':
    dic = count_length()
    print dic