import string
import re
import os


def read_from_dir(path='../Data/singleFile/'):
    pool = []
    dirs = [i for i in os.listdir(path) if i.find('cpp') != -1]
    for d in dirs:
        with open(os.path.join(path, d)) as f:
            pool.append(''.join(f.readlines()))
    return pool


def clearFile(f, pattern=None):
    if pattern is None:
        pattern = r'[^a-z0-9\"\':;><,.\/\?~`!@#$%^&*()_\+\-=\|\{\}\[\]]'
    return ''.join(re.split(pattern, f.lower()))


def winnowing(hv, window_size=4):
    if window_size > len(hv):
        raise ValueError('windows size must less than length of hash list')
    forb = set()
    fp = []
    for index in range(len(hv) - window_size + 1):
        t = [(hv[i], -i) for i in range(index, index + window_size)]
        t = sorted(t)
        if t[0][1] not in forb:
            fp.append(tuple(t[0]))
            forb.add(t[0][1])
        else:
            for j in t[1:]:
                if j[0] != t[0][0]:
                    break
                if j[1] not in forb:
                    forb.add(j[1])
                    fp.append(tuple(j))
    return fp


def rolling_hash(s, gram_size=4, hash_table=None):
    if hash_table is None:
        hash_table = ' {}{}{}'.format(string.ascii_lowercase, '0123456789', '"\':;><,.\/?~`!@#$%^&*()_+-=|{}[]')

    if len(s) < gram_size:
        raise ValueError('String length should larger than gram size')
    seq = []
    hv = 0
    n = 10 ** (gram_size - 1)
    for i in s[:gram_size]:
        h = hash_table.find(i)
        if h == -1:
            raise ValueError('Can not find in hash table')
        hv *= 10
        hv += h
    head = 0
    seq.append(hv)
    for i in s[gram_size:]:
        h = hash_table.find(i)
        if h == -1:
            raise ValueError('Can not find in hash table')
        hv -= n * hash_table.find(s[head])
        hv *= 10
        hv += h
        head += 1
        seq.append(hv)
    return seq


def get_hash_value(dat, hash_table=list(string.ascii_letters + ' '), gram_size=4):
    token = dat[:gram_size]
    fp = 0
    for t in token:
        fp *= 10
        try:
            fp += hash_table.index(t)
        except ValueError:
            fp = hash_table.__len__()
            hash_table.append(t)
    rst = [fp]
    prev_hv = hash_table.index(token[-1])
    for t in dat[gram_size:]:
        last_fp = int(rst[-1])
        last_fp -= gram_size * 10 * prev_hv
        last_fp *= 10
        try:
            last_fp += hash_table.index(t)
        except ValueError:
            last_fp += hash_table.__len__()
            hash_table.append(t)
        prev_hv = hash_table.index(t)
        rst.append(last_fp)
    return set(rst)


def sim(base, com):
    base = set(base)
    com = set(com)
    inter = base.intersection(com)
    score = (len(inter) / len(base), len(inter) / len(com))
    dist = 1 - (score[0] ** 2 / 2 + score[1] ** 2 / 2) ** 0.5
    return (score, dist)


def cal_sim_dup(fp_pool):
    sim_pool = {}
    for index, base in enumerate(fp_pool):
        for cindex, com in enumerate(fp_pool[index:]):
            if cindex == 0:
                continue
            sim_pool['{},{}'.format(index, index + cindex)] = sim(base, com)
    return sim_pool

if __name__ == '__main__':
    files = read_from_dir()
    files = [clearFile(f) for f in files]
    hv = [rolling_hash(f.lower) for f in files]
    fingerprint = [winnowing(i) for i in hv]
