Hi, I found in BiasAdd module, if I do not wrap self.bias with nn.Parameter, then I do not need to worry about retain_graph, but if I wrap it with nn.Parameer, it obligatory requires me to add retain_graph=True.
Do you know why? Thanks a lot!
class BiasAdd(nn.Module):

    def __init__(self,
                 channels,
                 opts,
                 act='linear', alpha=None, gain=None, lrmul=1):
        """
            BiasAdd
        """
        super(BiasAdd, self).__init__()

        self.opts = opts
        # fixme:
        # self.bias = nn.Parameter(torch.zeros(channels, 1, 1) * lrmul).to(opts.device) # error!
        self.bias = (torch.zeros(channels, 1, 1) * lrmul).to(opts.device) # ok

        self.act = act
        self.alpha = alpha if alpha is not None else 0.2
        self.gain = gain if gain is not None else 1.0

    def forward(self, x):
        # Pass Add bias.
        # x += self.bias (if self.bias wrap with nn.Parameter) ok!
        x = x + self.bias

        # Evaluate activation function.
        if self.act == "linear":
            pass
        elif self.act == 'lrelu':
            x = F.leaky_relu(x, self.alpha, inplace=True)
            x = x * np.sqrt(2)  # original repo def_gain=np.sqrt(2).

        # Scale by gain.
        if self.gain != 1:
            x = x * self.gain

        return x