Now I let BiasAdd like below and it works well without retain_graph = True.
class BiasAdd(nn.Module):

    def __init__(self,
                 channels,
                 opts,
                 act='linear', alpha=None, gain=None, lrmul=1):
        """
            BiasAdd
        """
        super(BiasAdd, self).__init__()

        self.opts = opts
        self.bias = torch.nn.Parameter((torch.zeros(channels, 1, 1) * lrmul))

    def forward(self, x):
       x += self.bias
       ...
       return x