class MyNet(nn.Module):
    def __init__(self):
        super(MyNet, self).__init__()
        self.conv1 = nn.Conv2d(3, 64, 3, padding=1)
        self.relu1 = nn.ReLU(True)
        self.conv2 = nn.Conv2d(64, 1, 3, padding=1)

    def forward(self, input):
        return self.conv2(self.relu1(self.conv1))

class MyNet2(nn.Module):
    def __init__(self):
        super(MyNet2, self).__init__()
        self.conv1 = nn.Conv2d(3, 64, 3, padding=1)
        self.relu1 = nn.ReLU(True)
     #  move this to forward
     #  self.conv2 = nn.Conv2d(64, 1, 3, padding=1)

    def forward(self, input):
        conv2_filters = torch.randn(1, 64, 3, 3)
        return F.conv2d(self.relu1(self.conv1)), conv2_filters, padding=1)

class MyNet3(nn.Module):
    def __init__(self):
        super(MyNet3, self).__init__()
        self.conv1 = nn.Conv2d(3, 64, 3, padding=1)
        self.relu1 = nn.ReLU(True)
     #  move this to forward
     #  self.conv2 = nn.Conv2d(64, 1, 3, padding=1)

    def forward(self, input):
        self.conv2_filters = torch.randn(1, 64, 3, 3)
        return F.conv2d(self.relu1(self.conv1)), self.conv2_filters, padding=1)

I suspect that optimizer can not get the parameters of conv2 in MyNet2 and MyNet3, then we must define the parameters in __init__ if we want them to get updated in optimizer.step() ?