I am trying to write a custom autograd.Function that solves logistic regression using LBFGS. While solve_logistic_regression(Xv,yv,.1) works flawlessly, lr(Xv,yv,.1) throws the following runtime error:
RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn
I’m not sure what is going on here.
The code:
import numpy as np

from torch.autograd import Function, Variable
import torch

N, n = 100, 2
X = np.random.randn(N, n)
y = np.random.randint(0,2,size=N)
Xv = Variable(torch.Tensor(X), requires_grad=False)
yv = Variable(torch.Tensor(y), requires_grad=False)

def solve_logistic_regression(X, y, lamb):
    N, n = X.shape
    theta = Variable(torch.ones(n), requires_grad=True)
    optimizer = torch.optim.LBFGS([theta], lr=.8)
    def closure():
        optimizer.zero_grad()
        pi = 1./(1.+torch.exp(-X.mm(theta.unsqueeze(-1))))
        loss = 1./N*torch.nn.BCELoss()(pi.squeeze(), y) + lamb/2*torch.norm(theta[:-1])**2
        print (loss.item())
        loss.backward()
        return loss
    optimizer.step(closure)
    return theta
    
class LogisticRegression(Function):
    @staticmethod
    def forward(ctx, X, y, lamb):
        theta = solve_logistic_regression(X, y, lamb)
        return theta
        
    @staticmethod
    def backward(ctx, grad_output):
        return None, None, None
lr = LogisticRegression.apply