Hello! I hope it’s ok I’m asking, but I’m having troubles with my nn code. It seems as if the loss dosen’t decrease and when running in debug mode I encounter this problem:
Exception has occurred: RuntimeError
one of the variables needed for gradient computation has been modified by an inplace operation: [torch.cuda.FloatTensor [10, 1]], which is output 0 of TBackward, is at version 2; expected version 1 instead. Hint: the backtrace further above shows the operation that failed to compute its gradient. The variable in question was changed in there or anywhere later. Good luck!
File “…/FinalProject/trip.py”, line 167, in 
loss.backward(retain_graph=True)
I’ve been searching and reading online answers and tried many things but can’t figure this out. I’d love to get your help.
The whole traceback:
[W python_anomaly_mode.cpp:104] Warning: Error detected in AddmmBackward. Traceback of forward call that caused the error:
File “…/anaconda3/lib/python3.8/runpy.py”, line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File “…/anaconda3/lib/python3.8/runpy.py”, line 87, in _run_code
exec(code, run_globals)
File “…/.vscode-server/extensions/ms-python.python-2021.9.1191016588/pythonFiles/lib/python/debugpy/main.py”, line 45, in 
cli.main()
File “/…/.vscode-server/extensions/ms-python.python-2021.9.1191016588/pythonFiles/lib/python/debugpy/…/debugpy/server/cli.py”, line 444, in main
run()
File “…/.vscode-server/extensions/ms-python.python-2021.9.1191016588/pythonFiles/lib/python/debugpy/…/debugpy/server/cli.py”, line 285, in run_file
runpy.run_path(target_as_str, run_name=compat.force_str(“main”))
File “/…/anaconda3/lib/python3.8/runpy.py”, line 265, in run_path
return _run_module_code(code, init_globals, run_name,
File “/…/anaconda3/lib/python3.8/runpy.py”, line 97, in _run_module_code
_run_code(code, mod_globals, init_globals,
File “/…/anaconda3/lib/python3.8/runpy.py”, line 87, in _run_code
exec(code, run_globals)
File “…/PycharmProjects/FinalProject/trip.py”, line 156, in 
out_q,out_p,out_n = model(query,pos,neg) # triplets and anchors into the nn
File “/…/anaconda3/lib/python3.8/site-packages/torch/nn/modules/module.py”, line 889, in _call_impl
result = self.forward(*input, **kwargs)
File “/…/PycharmProjects/FinalProject/trip.py”, line 94, in forward
out_n = self.net(neg)
File “/…/anaconda3/lib/python3.8/site-packages/torch/nn/modules/module.py”, line 889, in _call_impl
result = self.forward(*input, **kwargs)
File “/…/anaconda3/lib/python3.8/site-packages/torch/nn/modules/container.py”, line 119, in forward
input = module(input)
File “/…/anaconda3/lib/python3.8/site-packages/torch/nn/modules/module.py”, line 889, in _call_impl
result = self.forward(*input, **kwargs)
File “/…/anaconda3/lib/python3.8/site-packages/torch/nn/modules/linear.py”, line 94, in forward
return F.linear(input, self.weight, self.bias)
File “/…/anaconda3/lib/python3.8/site-packages/torch/nn/functional.py”, line 1753, in linear
return torch._C._nn.linear(input, weight, bias)
(function _print_stack)
Code attached here: It’s supposed to train on triplets I’ve generated and calculate triplet loss.

==================================

Neural Network

==================================

Define the Network Class

=========================
class MyNetwork(nn.Module):
def __init__(self):

    # call constructor from superclass

    super().__init__()

    # define network layers

    self.net = nn.Sequential(

        # Hidden Layer 1

        nn.Linear(num_of_features, 100),

        nn.ReLU(),

        # Hidden Layer 2

        nn.Linear(100, 10),

        nn.ReLU(),

        # Output Layer

        nn.Linear(10, 1)

    )

def forward(self, query, pos, neg):

    out_q = self.net(query)

    out_p = self.net(pos)

    out_n = self.net(neg)

    out_q = torch.clone(out_q)

    out_p = torch.clone(out_p)

    out_n = torch.clone(out_n)

    return out_q,out_p,out_n


Instantiate the model and send to cuda device:

---------------------------------------------
model = MyNetwork()
model.to(device)

Loss Criterion

================
triplet_loss_fn = nn.TripletMarginLoss(margin=1.0, p=2)

Optimization

=============

optimizer type:

---------------
optimizer = optim.SGD(model.parameters(), lr=lr, momentum=momentum, weight_decay=weight_decay)#,nesterov=True)

=========================

Train phase

=========================
params = list(model.parameters())
print(len(params))
train_loader = DataLoader(train_data, batch_size=batch_size,shuffle=True)
val_loader = DataLoader(val_data, batch_size=batch_size,shuffle=True)
train_losses, train_accuracy, val_losses, val_accuracy = ([] for i in range(4))  # create four empty lists
model.train()
triplet_train_losses=[]
epochs=20
print(“Starting Train Loop”)
for epoch in range(epochs):  # loop over the dataset multiple times
loss = 0.0

batches_acc = 0

triplets_num = 0

loss_tri = 0

loss_phy = 0

# iterate over the data

for batch_idx, data in enumerate(train_loader):

   # data = data.to(device)   # move data to the GPU (when using a GPU) 

    #features = (data[:,:,:-1].float()).to(device)

    features = torch.clone(data[:,:,:-1]).float().to(device)

    labels = torch.clone(data[:,0,-1]).to(device) 
    query = torch.clone(features[:,0,:]).to(device)
    pos = torch.clone(features[:,1,:]).to(device)
    neg = torch.clone(features[:,2,:]).to(device)

    

    # Forward pass:

    out_q,out_p,out_n = model(query,pos,neg) # triplets and anchors into the nn

    out_q = torch.clone(out_q).to(device)

    out_p = torch.clone(out_p).to(device)

    out_n = torch.clone(out_n).to(device)

    # compute loss

    loss = torch.clone(loss + triplet_loss_fn(out_q,out_p,out_n))

    # zero the parameter gradients

    optimizer.zero_grad()

    loss.backward(retain_graph=True)

    optimizer.step()

    params = list(model.parameters())   

    grads0=(list(model.parameters())[0].grad) 

    grads1=(list(model.parameters())[1].grad) 

    grads2=(list(model.parameters())[2].grad) 

    grads3=(list(model.parameters())[3].grad) 

    grads4=(list(model.parameters())[4].grad) 

    grads5=(list(model.parameters())[5].grad) 

    

# Normalizing the loss by the total number of train batches

num_batches = len(train_loader)

train_losses.append(loss/num_batches)


print("Epoch: {0} |loss: {1}% |".format(epoch+1,  train_losses[-1]))

Thank you very much!!!