i figured out how to use the chunk loader feature of pd.read_csv, but ran into difficulties since the iterator object (returned by read_csv with chunksize argument) can only draw samples at a fixed order (and i want the order to be shuffled after each epoch)
i found a way to bypass that, but i’m afraid it is still very slow. my new approach:


changed the sampler to a custom sampler such that each time getitem is called, it takes in a batch of indices


the getitem function takes the list of indices generated by the sampler and utilizes the skiprows feature of pd.read_csv.


so, to be clear, not using chunk loader. simply call read_csv everytime i draw a batch of samples and skip all rows but the rows of the batch.
i’m adding the code here, if anyone has a better solution please let me know (
import torch
import torch.nn as nn
from torch.utils.data import DataLoader
from preprocess import FacialKeypoints
import numpy as np
from torch.utils.data import Subset
from sklearn.model_selection import train_test_split as splitter
from helper_funcs import pixel_distance

transformed_dataset = FacialKeypoints(transform_vars={'is':True,'degrees':10,'flip_probability':0.5,'rotate_probability':0.5})
validation_dataset = FacialKeypoints(transform_vars={'is':False})
n = len(transformed_dataset)
num_train = int(np.ceil(len(transformed_dataset) * 0.85))
num_val = int(len(transformed_dataset) - num_train)

train_idx, val_idx = splitter(np.arange(n),train_size=num_train,shuffle=True)
batch_size = 16

trainset = Subset(transformed_dataset,train_idx)
valset = Subset(validation_dataset,val_idx)

train_sampler = torch.utils.data.sampler.BatchSampler(
    torch.utils.data.sampler.RandomSampler(trainset), batch_size=batch_size,drop_last=False)
val_sampler = torch.utils.data.sampler.BatchSampler(
    torch.utils.data.sampler.RandomSampler(valset), batch_size=batch_size,drop_last=False)

trainloader = DataLoader(trainset, sampler=train_sampler,num_workers=0)
valoader = DataLoader(valset, sampler=val_sampler, num_workers=0)

if torch.cuda.is_available():
    device = torch.device('cuda:0')


model2 = nn.Sequential(
    nn.Conv2d(1,32,3),
    nn.MaxPool2d(kernel_size=5,stride=2),
    nn.ReLU(),
    nn.Conv2d(32,64,2),
    nn.MaxPool2d(kernel_size=5,stride=2),
    nn.ReLU(),
    nn.Dropout2d(0.5),
    nn.Conv2d(64,128,2),
    nn.MaxPool2d(kernel_size=3,stride=2),
    nn.ReLU(),
    nn.Flatten(),
    nn.Linear(10368,500),
    nn.ReLU(),
    nn.Dropout(0.5),
    nn.Linear(500,500),
    nn.ReLU(),
    nn.Linear(500,30)
)

model2.to(device)
total_loss = {'train':[],'val':[]}
criterion2 = nn.MSELoss()
optimizer2 = torch.optim.Adam(model2.parameters(),lr=0.001)


total_loss = {'train':[],'val':[]}
for epoch in range(10):
    pixel_dist = []
    print('in epoch {}/100 :'.format(epoch+1))
    for sample in trainloader:
        losses = []
        input = sample['image'].squeeze(0).to(dtype = torch.float,device = device)
        batch = input.shape[0]
        target = sample['keypoints'].squeeze(0).to(dtype = torch.float,device = device)
        optimizer2.zero_grad()
        output = model2(input)
        pixel_dist.append(pixel_distance(output, target))
        loss2 = criterion2(output,target)
        loss2.backward()
        optimizer2.step()
        losses.append(loss2.data.item())
    a = np.mean(losses)
    total_loss['train'].append(a)
    print('train loss = {}'.format(a))
    print('avg training pixel distance = {}'.format(np.mean(pixel_dist)))
    pixel_dist = []
    for sample in valoader:
        with torch.no_grad():
            losses = []
            input = sample['image'].squeeze(0).to(dtype = torch.float,device = device)
            batch = input.shape[0]
            target = sample['keypoints'].squeeze(0).to(dtype = torch.float,device = device)
            output = model2(input)
            pixel_dist.append(pixel_distance(output,target))
            loss2 = criterion2(output, target)
            losses.append(loss2.data.item())
    a = np.mean(losses)
    print('validation loss = {}'.format(a))
    total_loss['val'].append(a)
    print('avg validatioin pixel distance = {}'.format(np.mean(pixel_dist)))

'''def check_sample(loader=valoader,model=model2,device=device):
device2 = torch.device('cpu')
plots = 16//3
x = next(iter(loader))
y_true = x['keypoints']
y_true = y_true.reshape(16,15,2)
x = x['image'].to(device)
x = x.view(16,1,96,96)
y = model(x)
y = y.reshape(16,15,2).to(device2)
x = x.to(device2)

fig,ax = plt.subplots(3,plots)
k=0
for i in range(plots):
    for j in range(3):
        ax[i,j].scatter(y_true[k,:,0].detach().numpy(),y_true[k,:,1].detach().numpy())
        ax[i,j].imshow(x[k].squeeze())
        ax[i,j].scatter(y[k,:,0].detach().numpy(),y[k,:,1].detach().numpy())
        k=k+1
plt.show()
'''```


def __getitem__(self, idx):
    self.idx = idx
    if torch.is_tensor(idx):
        idx = idx.tolist()
    image = torch.from_numpy(np.array(read_csv('data/Q3/training_images.csv',header=None,
                     skiprows=np.delete(self.indices,self.idx))))
    keypoints = torch.from_numpy(np.array(read_csv('data/Q3/training_labels.csv',header=None,
                     skiprows=np.delete(self.indices,self.idx))))
    if self.transform_vars['is']:
        image, keypoints = self.transform(image, keypoints)
        return {'image':image, 'keypoints':keypoints}
    else:
        image = image.reshape((-1,96, 96))
        return {'image':image.unsqueeze(1),'keypoints':keypoints}