I have a dataset of about 40 Gb. I have saved this as .npz files (150 files) batch0.npz to batch150.npz. Each contains 64 data points (batch size = 64). Need to create a balance between I/O reads hence saved 64 data points but I want to train my model with batch size = 32. with the shuffle. How should I do this? Below is the code I have right now.
ai = np.random.randn(64,70,128)
aj = np.random.randn(64,70,18,128)
ak = tnp.random.randn(64,70,50,128)
al = torch.rand(64,70,60,128)

sample = {ai, aj, ak, al}

np.savez("batch0.npz", "ai"=ai, "aj"=aj, "ak"=ak, "al"=al)

I create a dataset as follows
class load_batch_files(Dataset):
    def __init__(self):
        self.data_files = os.listdir("batch_data/")
        self.data_files.sort()

    def __getitem__(self, idx):
        # pdb.set_trace()
        return np.load(self.data_files[idx])

    def __len__(self):
        return len(self.data_files)

def somecollate(batch):
    # pdb.set_trace()
    batch = batch[0]
    ai = torch.tensor(batch["ai"]).float().to(device)
    aj = torch.tensor(batch["aj"]).float().to(device)
    ak = torch.tensor(batch["ak"]).float().to(device)
    al = torch.tensor(batch["al"]).float().to(device)
    batch = {"ai": ai, "aj": aj, "ak": ak, "al": al}
    return batch

desc_train = load_batch_files()
trainloader = DataLoader(desc_train, batch_size=32, collate_fn=somecollate)

This does not give me data with a batch size of 32. even shuffle does not work. what am I doing wrong?
This is what I have tried loading-huge-data-functionality 15