hello everyone,
if someone can help me
my problem is that i’m using kfols method to do the cross validation, which means that i have 5 itération to fit my data into training and testing set
i made a loop for around my data, when i do just a training it works normally but when i do the validation after that, i get this error RuntimeError: CUDA error: an illegal memory access was encountered
knowing that i tryed

device = torch.device(‘cuda’ if torch.cuda.is_available() else ‘cpu’)
print(‘Using device:’, device)
print()
#Additional Info when using cuda
if device.type == ‘cuda’:
print(torch.cuda.get_device_name(0))
print(‘Memory Usage:’)
print(‘Allocated:’, round(torch.cuda.memory_allocated(0)/10243,1), ‘GB’)
print('Cached:   ', round(torch.cuda.memory_cached(0)/10243,1), ‘GB’)

and i get

Using device: cuda
GeForce GTX 1080 Ti
Memory Usage:
Allocated: 0.4 GB
Cached:    1.9 GB

here is my code :

for l in range(0,len(XTrain)):
loss=0.0
loss_test=0.0

Net=UNet(1,4)  
Net.apply(init_weights)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# torch.backends.cudnn.benchmark = True
# torch.cuda.set_device(0)
Net.to(device)
size=[]
size_test=[]
taille=[]
x={}
y={}
z={}
x_test={}
y_test={}
z_test={}
data_train_norm=[]
data_test_norm=[]
gt_train_norm=[]
gt_test_norm=[]

for k in range(0,int(0.8*len(data_ed))):
    data_train_norm.append(Preprocessing(XTrain[l][k]))
    gt_train_norm.append((YTrain[l][k]))
    size.append(data_train_norm[k].shape)
    x[k],y[k],z[k]=size[k]
    
for k in range(0,int(0.2*len(data_ed))):
    
    data_test_norm.append(Preprocessing(XTest[l][k]))
    gt_test_norm.append((YTest[l][k]))
    size_test.append(data_test_norm[k].shape)
    x_test[k],y_test[k],z_test[k]=size_test[k]

    
    
image_resized_train=[]
image_resized_test=[]
image_resized_train_gt=[]
image_resized_test_gt=[]


for i in range(0,int(0.8*len(data_ed))):
    for j in range(z[i]):
        
        image_resized_train.append(resize_image(data_train_norm[i][:,:,j],H_train[l][i][1],1.25))
        image_resized_train_gt.append(resize_gt(gt_train_norm[i][:,:,j],H_train[l][i][1],1.25))
      

for i in range(0,int(0.2*len(data_ed))):
    for j in range(z_test[i]):
        
        image_resized_test.append(resize_image(data_test_norm[i][:,:,j],H_test[l][i][1],1.25))
        image_resized_test_gt.append(resize_gt(gt_test_norm[i][:,:,j],H_test[l][i][1],1.25))
final_train=[] 
final_test=[]
final_train_gt=[] 
final_test_gt=[]
img_train=[]  
img_test=[]
img_train_gt=[]  
img_test_gt=[]
width=557
height=667
data_tensor_test=[]
data_tensor_train=[]
gt_tensor_test=[]
gt_tensor_train=[]

for i in range(0,len(image_resized_train)):  
    
    img_train.append(Resize(image_resized_train[i],width,height))
    
    final_train.append(crop(img_train[i],256,256))

    data_tensor_train.append(torch.from_numpy(np.array(final_train[i])).float())
    # groudtruth train 
    
    img_train_gt.append(Resize(image_resized_train_gt[i],width,height))
    
    final_train_gt.append(crop(img_train_gt[i],256,256))

    gt_tensor_train.append(torch.from_numpy(np.array(final_train_gt[i])).float())
    
      
for i in range(0,len(image_resized_test)):
    
    img_test.append(Resize(image_resized_test[i],width,height))
    
    final_test.append(crop(img_test[i],256,256))

    data_tensor_test.append(torch.from_numpy(np.array(final_test[i])).float())
    
    img_test_gt.append(Resize(image_resized_test_gt[i],width,height))
    
    final_test_gt.append(crop(img_test_gt[i],256,256))

    gt_tensor_test.append(torch.from_numpy(np.array(final_test_gt[i])).float())

    
X_train=torch.stack(data_tensor_train)
X_test=torch.stack(data_tensor_test)
Y_train=torch.stack(gt_tensor_train)
Y_test=torch.stack(gt_tensor_test)
print(l,X_train.shape,X_test.shape,Y_train.shape,Y_test.shape)

class LoadDataset_train(Dataset):
    

    def __init__(self):
        # load data
        self.X=X_train
        self.Y=Y_train
        self.len=X_train.shape[0]
        

    def __len__(self):
        
        # dataset's size
        return self.len
        

    def __getitem__(self, idx):
        X,Y = self.X[idx], self.Y[idx]
        # the position
        X = X.view(1,X.shape[0],X.shape[1])
        sample=X,Y
        return sample 
    
class LoadDataset_test(Dataset):
    

    def __init__(self):
        # load data
        self.X=X_test
        self.Y=Y_test
        self.len=X_test.shape[0]
        

    def __len__(self):
        
        # dataset's size
        return self.len
        

    def __getitem__(self, idx):
        X,Y = self.X[idx], self.Y[idx]
        # the position
        X = X.view(1,X.shape[0],X.shape[1])
        sample=X,Y
        return sample       
    
dataset=LoadDataset_train()   
trainloader = torch.utils.data.DataLoader(dataset=dataset, batch_size=4, shuffle=True, num_workers=1)
dataset_test=LoadDataset_test()       
trainloader_test = torch.utils.data.DataLoader(dataset=dataset_test, batch_size=1, shuffle=True, num_workers=1)


# print(trainloader)

learning_rate=10e-3
#define the weights using in the loss function 
weights = torch.tensor([0.07,0.31,0.31,0.31]).to(device)  ########
#define a loss function
criterion = nn.CrossEntropyLoss(weights)  
#define the optimizer aDAM 
optimizer=optim.Adam(Net.parameters(), lr=learning_rate , weight_decay=10e-8)



n_epochs=10
loss_values=[]
loss_values_test=[]

for epoch in range(n_epochs):
    Net.train()
    running_loss = 0.0
  
    
    
    
    for i, data in enumerate(trainloader):
        
            
        # get the inputs; data is a list of [inputs, labels]

        inputs, labels = data[0].to(device), data[1].to(device).long()
        
        
            
        optimizer.zero_grad()
        outputs=Net(inputs)
        loss=criterion(outputs,labels)
        running_loss+=loss.item()
        loss.backward()
        optimizer.step()
        
       
        print(i)
    
    loss_values.append(running_loss/len(trainloader)) 
     
    running_loss_test=0.0
    
    
    Net.eval()
    with torch.no_grad():
        
        for j, data_test in enumerate(trainloader_test,0):
        
    
            inputs_test, labels_test = data_test[0].to(device), data_test[1].to(device).long()
            outputs_test=Net(inputs_test)
            loss_test=criterion(outputs_test,labels_test)
            running_loss_test+=loss_test.item()
            

    loss_values_test.append(running_loss_test/len(trainloader_test))


THANK YOU !!!