text_plain_output_2.png text_plain_output_1.png
import itertools

import numpy as np # linear algebra

import random

"""
create sample data
user and product are segmented
"""
random.seed(0)
NN_word = 2000
NN_sentence = 10000
NN_SEG = 7
class Seq(object):
    def __init__(self, neg_sample=5, batch_size=32, stop=None, common_prods=[50, 100, 200, 500, 1000]):
        self.common_prods = common_prods
        self.batch_size = batch_size
        self.product_list = [ee + 1 for ee in range(NN_word)]
        self.product_set = set(self.product_list)
        self.neg_sample = neg_sample
        nn_each_group, amari = divmod(len(self.product_list), NN_SEG)
        if amari != 0:
            nn_each_group += 1
        del_none = lambda l: filter(lambda x: x is not None, l)
        self.product_group = [[e1 for e1 in del_none(ee)] for ee in itertools.zip_longest(*[iter(self.product_list)] * nn_each_group)]
        self.user_list = [ee + 1 for ee in range(NN_sentence)]
        self.user_list_org = [ee for ee in self.user_list]
        if stop is not None:
            self.user_list = self.user_list[:stop]
        self.user_set = set(self.user_list)
        self.user_id_next = self.user_list[0]
        self.create_seed()
        'estimate self length'
        self.initialize_it()
        self.len = 1
        for _ in self.it:
            self.len += 1
        self.initialize_it()
    def initialize_it(self):
        self.it = iter(range(0, len(self.user_list), self.batch_size))
        self.idx_next = self.it.__next__()
    def create_seed(self):
        self.seeds = {}
        for ii, user_id in enumerate(self.user_list):
            self.seeds[user_id] = ii
    def __len__(self):
        return self.len
    def __iter__(self):
        return self
    def __next__(self):
        idx = self.idx_next
        self.user_ids_part = self.user_list[idx:idx + self.batch_size if idx + self.batch_size < len(self.user_list) else len(self.user_list)]
        res = self.getpart(self.user_ids_part)
        try:
            self.idx_next = self.it.__next__()
        except StopIteration:
            self.initialize_it()
        return res
    def __getitem__(self, user_id):
        ret_users, ret_prods, ret_y = self.get_data(user_id)
        return ({'input_user': np.array(ret_users), 'input_prod': np.array(ret_prods)}, ret_y)
    def get_data(self, user_id):
        random.seed(self.seeds[user_id])
        nword = random.randint(5, 20)
        a, _ = divmod(len(self.user_list_org), NN_SEG)
        ii = int(user_id / (a + 1))
        prods = random.sample(self.product_group[ii], nword)
        prods.extend(self.common_prods)
        prods = list(set(prods))
        neg = self.get_neg(prods)
        ret_users = [user_id] * (len(prods) * (1 + self.neg_sample))
        ret_prods = prods + neg
        ret_y = [1] * len(prods) + [0] * len(neg)
        return (ret_users, ret_prods, ret_y)
    def get_neg(self, prods):
        o = self.product_set.difference(prods)
        random.seed()
        neg = random.sample(o, len(prods) * self.neg_sample)
        return neg
    def getpart(self, user_ids_part):
        x_input_user = []
        x_input_prod = []
        y = []
        for user_id in user_ids_part:
            x_train, y_train = self[user_id]
            x_input_prod.extend(x_train['input_prod'].tolist())
            x_input_user.extend(x_train['input_user'].tolist())
            y.append(y_train)
        return ({'input_prod': np.array(x_input_prod), 'input_user': np.array(x_input_user)}, np.concatenate(y))

seq = Seq(neg_sample=1, common_prods=[])
print(len(seq))
X_list = [np.zeros((1, NN_word + 1))]
for iuser in seq.user_list:
    x_train, y_train = seq[iuser]
    prods = x_train['input_prod'][np.array(y_train) == 1]
    irow = np.zeros((1, NN_word + 1))
    irow[0, prods] = 1
    X_list.append(irow)
X = np.concatenate(X_list)
print(X.shape)
X