I wanted to try out different sets of optimizer hyperparameters for each element of a tensor. I tried the following but I am getting a non-leaf tensor error, possibly because I am indexing the tensor:
y = torch.ones(3)
y.requires_grad_()
print(y)
opt2 = torch.optim.SGD([{'params':[y[0]],'lr':0.1},{'params':[y[1]],'lr':1},{'params':[y[2]],'lr':10}])
loss2 = y.sum()
opt2.zero_grad()
loss2.backward()
opt2.step()
print(y)


ValueError                                Traceback (most recent call last)
 in ()
2 y.requires_grad_()
3 print(y)
----> 4 opt2 = torch.optim.SGD([{‘params’:[y[0]],‘lr’:0.1},{‘params’:[y[1]],‘lr’:1},{‘params’:[y[2]],‘lr’:10}])
5 loss2 = y.sum()
6 opt2.zero_grad()


/usr/local/lib/python3.6/dist-packages/torch/optim/sgd.py in init(self, params, lr, momentum, dampening, weight_decay, nesterov)
62         if nesterov and (momentum <= 0 or dampening != 0):
63             raise ValueError(“Nesterov momentum requires a momentum and zero dampening”)
—> 64         super(SGD, self).init(params, defaults)
65
66     def setstate(self, state):


/usr/local/lib/python3.6/dist-packages/torch/optim/optimizer.py in init(self, params, defaults)
41
42         for param_group in param_groups:
—> 43             self.add_param_group(param_group)
44
45     def getstate(self):


/usr/local/lib/python3.6/dist-packages/torch/optim/optimizer.py in add_param_group(self, param_group)
191                                 "but one of the params is " + torch.typename(param))
192             if not param.is_leaf:
–> 193                 raise ValueError(“can’t optimize a non-leaf Tensor”)
194
195         for name, default in self.defaults.items():


ValueError: can’t optimize a non-leaf Tensor