text_plain_output_1.png
from tqdm import tqdm

import itertools

import numpy as np

import pandas as pd

import pandas as pd
data = pd.read_csv('/kaggle/input/docs-for-lsh/docs_for_lsh.csv')

import numpy as np
data = data.values[:, 1:]
np.random.seed(0)
n = 30
m = data.shape[0]
length = data.shape[1]
signMatrix = np.zeros((n, m))

def bucketAllocate(r, b):
    Bucket = {}
    for i in range(b):
        tmp = signMatrix[b:b + r]
        for j in range(m):
            signBand = list(tmp[:, j])
            signBand.append(b)
            hashValue = hash(str(signBand))
            if hashValue not in Bucket:
                Bucket[hashValue] = [j]
            else:
                Bucket[hashValue].append(j)
    return Bucket

r = 30
b = 1
bucket = bucketAllocate(r, b)

def Jaccard(i, j):
    iData = data[i, :]
    jData = data[j, :]
    count1 = 0
    count2 = 0
    for k in range(length):
        if max(iData[k], jData[k]):
            count1 += 1
        if min(iData[k], jData[k]):
            count2 += 1
    return count2 / count1
Jaccard(1, 2)

import itertools
from tqdm import tqdm
MaxSimilarity = 0
SimilarPair = None
for obj in tqdm(bucket.values()):
    for p in itertools.combinations(obj, 2):
        jaccardSim = Jaccard(p[0], p[1])
        if jaccardSim > MaxSimilarity:
            MaxSimilarity = jaccardSim
            SimilarPair = p

def bucketFind(r, b):
    Bucket = {}
    hash0 = 0
    for i in range(b):
        tmp = signMatrix[b:b + r]
        for j in range(m):
            signBand = list(tmp[:, j])
            signBand.append(b)
            hashValue = hash(str(signBand))
            if j == 0:
                hash0 = hashValue
            elif hashValue == hash0:
                Bucket[j] = Jaccard(0, j)
    return Bucket

r = 10
b = 3
bucket = bucketFind(r, b)
print(bucket)