I used the ‘get_data_user_sep’ function and ‘UserSepDataset’ class to get the data.
This is my data size.
image788×96 35.3 KB
from dataset.dataset_user_sep import UserSepDataset

def run(i):
    """
    i: single integer represents dataset number
    """
    user_base_path = '{}/{}/processed'.format(ARGS.base_path, ARGS.dataset_name)

    train_data_path = '{}/{}/train/'.format(user_base_path, i)
    val_data_path = '{}/{}/val/'.format(user_base_path, i)
    test_data_path = '{}/{}/test/'.format(user_base_path, i)

    # 추가
    print('Run...')
    print(train_data_path)

    train_sample_infos, num_of_train_user = util.get_data_user_sep(train_data_path)
    val_sample_infos, num_of_val_user = util.get_data_user_sep(val_data_path)
    test_sample_infos, num_of_test_user = util.get_data_user_sep(test_data_path)

    # 추가
    print('End reading...')

    train_data = UserSepDataset('train', train_sample_infos, ARGS.dataset_name)
    val_data = UserSepDataset('val', val_sample_infos, ARGS.dataset_name)
    test_data = UserSepDataset('test', test_sample_infos, ARGS.dataset_name)

    #print(f'Train: # of users: {num_of_train_user}, # of samples: {len(train_sample_infos)}')
    #print(f'Validation: # of users: {num_of_val_user}, # of samples: {len(val_sample_infos)}')
    #print(f'Test: # of users: {num_of_test_user}, # of samples: {len(test_sample_infos)}')

    # 추가
    print('Get Model...')
    model, d_model = get_model()

    # 추가
    print('Making trainer...!!')
    trainer = Trainer(model, ARGS.device, ARGS.warm_up_step_count,
                      d_model, ARGS.num_epochs, ARGS.weight_path,
                      ARGS.lr, train_data, val_data, test_data)
    print('Start training...')
    trainer.train()
    trainer.test(0)
    return trainer.test_acc, trainer.test_auc


This is my get_data_user_sep function!
def get_data_user_sep(data_path):
    # almost same as get_sample_info
    # for user separated format data
    sample_infos = []

    # 추가
    print('data_path: ', data_path)

    # get list of all files
    user_path_list = os.listdir(data_path)
    num_of_users = len(user_path_list)

    # 추가
    print('Start reading lines... ')

    i = 1
    for user_path in user_path_list:
        i += 1
        if i%10000 == 0:
            print(i)
        with open(data_path + user_path, 'rb') as f:
            lines = f.readlines()
            lines = lines[1:]
            num_of_interactions = len(lines)
            for end_index in range(num_of_interactions):
                sample_infos.append((data_path + user_path, end_index))

    return sample_infos, num_of_users

This is my UserSepDataset class.
import torch
from torch.utils.data import Dataset
from config import ARGS
from constant import *


class UserSepDataset(Dataset):

    def __init__(self, name, sample_infos, dataset_name='ASSISTments2009'):
        self._name = name # train, val, test
        self._sample_infos = sample_infos # list of (user_path, target_index)
        self._dataset_name = dataset_name

    def get_sequence(self, sample):
        user_path, target_index = sample
        with open(user_path, 'r') as f:
            data = f.readlines()[1:] # header exists
            data = data[:target_index+1]
            user_data_length = len(data)

        if user_data_length > ARGS.seq_size + 1:
            data = data[-(ARGS.seq_size + 1):]
            pad_counts = 0
        else:
            pad_counts = ARGS.seq_size + 1 - user_data_length

        input_list = []
        for idx, line in enumerate(data):
            line = line.rstrip().split(',')
            tag_id = int(line[0])
            is_correct = int(line[1])

            if idx == len(data) - 1:
                last_is_correct = is_correct
                target_id = tag_id
            else:
                if is_correct:
                    input_list.append(tag_id)
                else:
                    input_list.append(tag_id + QUESTION_NUM[self._dataset_name])

        paddings = [PAD_INDEX] * pad_counts
        input_list = paddings + input_list
        assert len(input_list) == ARGS.seq_size, "sequence size error"

        return {
            'label': torch.Tensor([last_is_correct]).long(),
            'input': torch.Tensor(input_list).long(),
            'target_id': torch.Tensor([target_id - 1]).long()
        }

    def __repr__(self):
        return '{}: # of samples: {}'.format(self._name, len(self._sample_infos))

    def __len__(self):
        return len(self._sample_infos)

    def __getitem__(self, index):
        return self.get_sequence(self._sample_infos[index])