I’m dealing with MRI data, and I converted these files into numpy files and saved them.
my data consisted of input : [3600, 512, 512] (N,H,W) -1.8G and mask : [3600,8,512,512] (N,classes,H,W)-28.125G.
I’m using U-Net for segmentation and GPU is ‘NVIDIA GeForce RTX 2080Ti’ with 18G ram.
Data has been loaded as follows:
class trainDataset(torch.utils.data.Dataset):
    def __init__(self, data, target, transform=None):
        self.data = data.astype(np.float32)
        self.data = normalize(data)
        self.target = target.astype(np.float32)
        self.transform = transform
        
    def __getitem__(self, index):
        x = self.data[index]
        y = self.target[index]
        
        if self.transform:
            x = self.transform(x)
            
        return x, y
    
    def __len__(self):
        return len(self.data)

numpy_data = np.load(image_path+'MRtrain.npy')
numpy_target = np.load(mask_path+'RStrain.npy')

traindataset = trainDataset(numpy_data, numpy_target ,transform = transform)         

trainloader = torch.utils.data.DataLoader(traindataset, batch_size = batch_size, shuffle=True, num_workers=0, pin_memory=False)


and train loop as follow :
def fit(epoch,model,data_loader,phase='train',volatile=False):
    if phase == 'train':
        exp_lr_scheduler.step()
        model.train()
    if phase == 'valid':
        model.eval()
    running_loss = 0.0
    for batch_idx , (data,target) in enumerate(data_loader):
        inputs,target = data.cpu(),target.cpu()
        if is_cuda:
            inputs,target = data.cuda(),target.cuda()
        inputs , target = Variable(inputs),Variable(target)
        if phase == 'train':
            optimizer.zero_grad()
            
        output = model(inputs)
        pred = torch.sigmoid(output)
        loss = dice(pred,target)

        running_loss += loss.data.item()
        
        if phase == 'train':
            loss.backward()
            optimizer.step()
    
    loss = running_loss/len(data_loader.dataset)
    
    print('{} Dice_Loss: {:.4f}'.format(
                phase, loss))
    return loss

Training to 515x515 size will cause cuda memory error.
I think it’s inefficient to deal with numpy file, but is there a good way?