thanks. I’ve made some progress but my setup doesn’t look to be iterating properly at the moment. the code doesn’t hit my print lines in the enumerate stage.
here’s my code:
import torch
from torch.utils import data
import pandas as pd

class MyDataset(data.Dataset):
    def __init__(self, csv_path, chunkSize):
        self.chunksize = chunkSize
        self.reader = pd.read_csv(csv_path, sep=',', chunksize=self.chunksize, header=None, iterator=True)
    
    def __len__(self):
        return self.chunksize

    def __getitem__(self, index):
        data = self.reader.get_chunk(self.chunksize)
        tensorData = torch.as_tensor(data.values, dtype=torch.float32)
        inputs = tensorData[:, :-1]
        labels = tensorData[:, 99]
        return inputs, labels 

def main():
    batch_size = 100

    kwargs = {}
    custom_data_from_csv = MyDataset('data/mydata.txt', batch_size)
    train_loader = data.DataLoader(dataset=custom_data_from_csv, batch_size=batch_size, shuffle=True, **kwargs)
            
    for inputData, target in enumerate(train_loader):
        print(inputData)
        print(target)

if __name__ == '__main__':
    main()

also i wasn’t sure if len was meant to be the size of each chunk or the total number of chunks in the entire file.