my network is running slow and i suspect it’s got to do with the loading scheme, which is based on tutorials and simply loads all data to the dataset object during initialization.
am i right to think that it might help to load a chunk of data at each time, iterate through this chunk, update weights and then discard the chunk from memory and start again with another chunk?
i found [How to use dataset larger than memory? 9] (this link) to be close to what i’m looking for but it ended with no straight up solution.
my current dataest class, shown below, is initialized by loading the entire dataset and performing some procedures on it.
storing the entire preprocessed data as a variable of the class instance.
the get item method receives a random idx and returns the sample (input and target) after some more transformations that i apply manually (in the getitem method).
i’m not sure how to continue from here because if i set read_csv to “chunk mode” it returns an iterator. this way how do i perform preprocessing in the initialization?
i could preprocess in the getitem method, but then get item would get a chunk of data instead of a single sample, how can i pass that?
i’m adding my current dataset for your consideration. generally any advice would be appreciated but what i couldn’t, and would love to find is a sipmle example for my case: full implementation of loading chunks to a dataset object and configuring it with a dataloader object and all the way to a training loop.
‘’’
class FacialKeypoints(Dataset):
def __init__(self, test=False, cols=None,FTRAIN = 'data/Q3/training.csv', FTEST = 'EX1/Q3/test.csv', transform_vars=None, batch_size = 16):
    fname = FTEST if test else FTRAIN
    df = read_csv(os.path.expanduser(fname))  # load pandas dataframe

    # The Image column has pixel values separated by space; convert
    # the values to numpy arrays:
    df['Image'] = df['Image'].apply(lambda im: np.fromstring(im, sep=' '))

    print('number of values in each column: ', df.count())  # prints the number of values for each column
    df = df.dropna()  # drop all rows that have missing values in them
    X = np.vstack(df['Image'].values) / 255.  # scale pixel values to [0, 1]
    X = X.astype(np.float32)
    image_size = int(np.sqrt(X.shape[1]))
    Y = []
    if not test:  # only FTRAIN has any target columns
        y = df[df.columns[:-1]].values
        y2 = y.reshape(y.shape[0],15,2)
        for coords in y2:
            mask = np.zeros((image_size,image_size))
            for pair in coords:
                pair = pair.round().astype(int)
                mask[pair[1] - 1, pair[0] - 1] += 1
                if mask[pair[1] - 1, pair[0] - 1]==3:
                    print('bad')
            Y.append(mask)
        Y = np.array(Y)
        y = (y - 48) / 48  # scale target coordinates to [-1, 1]
        X, y, Y = shuffle(X, y, Y, random_state=42)  # shuffle train data
        y = y.astype(np.float32)
    else:
        y = None

    self.X = torch.tensor(X,dtype=torch.float32)
    self.transform_vars = transform_vars
    self.y = torch.tensor(y)
    self.Y = torch.tensor(Y,dtype=torch.float32)
    print('finished loading')
def __len__(self):
    return len(self.X)

def transform(self,image, mask):
    image = image.reshape(96,96)
    flip_prob = self.transform_vars['flip_probability']
    rotate_prob = self.transform_vars['rotate_probability']

    if torch.rand(1)>flip_prob:
        image = TF.hflip(image)
        mask_points = torch.nonzero(mask, as_tuple=False)
        newmask = torch.zeros((96, 96))
        for pair in mask_points:
            newpair = flip(pair)
            newmask[newpair[0] - 1, newpair[1] - 1] = mask[pair[0],pair[1]]
        mask = newmask
        #mask = TF.hflip(mask)
    if torch.rand(1)<rotate_prob:
        avg_pixel = image.mean()
        degrees = self.transform_vars['degrees']
        deg = int(torch.rand(1).item() * degrees - degrees)
        image_r = TF.to_tensor(TF.rotate(TF.to_pil_image(image),deg)).squeeze()
        image_r[(image_r==0) * (image!=0)] = avg_pixel
        image = image_r
        mask_points = torch.nonzero(mask,as_tuple=False)
        newmask = torch.zeros((96,96))
        for pair in mask_points:
            newpair = tilt(pair,deg)
            newmask[newpair[0] - 1,newpair[1] - 1] = mask[pair[0],pair[1]]
        mask = newmask
        #mask = TF.to_pil_image(mask)
        #mask = TF.rotate(mask, deg,resample=PIL.Image.NEAREST)
        #mask = TF.to_tensor(mask).squeeze()
        #mask = TF.to_tensor(TF.rotate(TF.to_pil_image(mask), deg)).squeeze()
    return image.unsqueeze(0), mask

def update_target(self,mask):
    keypoints = (mask==1).nonzero(as_tuple=False).reshape(-1)
    keypoints = torch.hstack([keypoints,(mask==2).nonzero(as_tuple=False).reshape(-1).repeat(2)])
    keypoints = torch.from_numpy((keypoints.numpy() - 48) / 48)
    if keypoints.shape[0]!=30:
        print('bad after transform')
        temp = self.__getitem__(torch.randint(0,len(self),[1]).item())
        return temp['keypoints']
    else:
        return keypoints 

def __getitem__(self, idx):
    self.idx = idx
    if torch.is_tensor(idx):
        idx = idx.tolist()
    image = self.X[idx]
    keypoints = self.y[idx]
    mask = self.Y[idx]
    if self.transform_vars['is']:
        image, mask = self.transform(image, mask)
        keypoints = self.update_target(mask).to(dtype = torch.float32)
        return {'image':image, 'keypoints':keypoints}
    else:
        return {'image':image,'keypoints':keypoints}

‘’’