I guess there are too many steps to do something reproducible but the only part where I use the model is this (the problem happes when I execute this for the second time):
model = Hyper3DNetLiteReg(img_shape=(1, nbands, windowSize, windowSize))
modelcrf = CRFlayer(nodes=windowSize ** 2)
model.to(device)
modelcrf.to(device)

# Training parameters
criterion = nn.MSELoss()
params = list(model.parameters())  # CNN parameters
params.extend(list(modelcrf.parameters()))  # Pairwise parameters
optimizer2 = optim.Adadelta(params, lr=1.0)


for epoch in range(epochs):  # Epoch loop
    model.network.train()  # Sets training mode
    model.crf.train()
    running_loss = 0.0
    for step in range(T):  # Batch loop
        # Generate indexes of the batch
        inds = indexes[step * batch_size:(step + 1) * batch_size]
        trainb = train[inds]
        # Get actual batches
        trainxb = torch.from_numpy(trainx[inds]).float().to(device)
        trainyb = torch.from_numpy(
        np.reshape(train_y[trainb],
                   (train_y[trainb].shape[0], 1,
                    train_y[trainb].shape[1] * train_y[trainb].shape[2]))).float().to(device)
        # zero the parameter gradients
        model.optimizer2.zero_grad()
        # forward + backward + optimize
        outputs = model.network(trainxb)
        outputs = torch.reshape(outputs, (outputs.shape[0], 1, outputs.shape[1] * outputs.shape[2]))
        loss = model.crf(outputs, trainyb, self.Lh, self.Lv, self.Ld1, self.Ld2, device, lossC=True)
        loss.backward()
        model.optimizer2.step()