I was about to implement a network consisting of residual connections, and encountered an error RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation.
I read about this thread and dig deeper into the implementation of residual connections using the two implementations below.
# implementation 1
class ResidualUnitA(nn.Module):
    def __init__(self):
        super(ResidualUnitA, self).__init__()
        self.conv_0 = nn.Conv2d(3, 24, 3, padding=1, bias=False)
        self.bn_0 = nn.BatchNorm2d(24)
        self.conv_1 = nn.Conv2d(24, 24, 3, padding=1, bias=False)
        self.bn_1 = nn.BatchNorm2d(24)

    def forward(self, x):
        x = self.conv_0(x)
        x = self.bn_0(x)
        x = F.relu(x, inplace=True)

        residual = self.conv_1(x)
        residual = self.bn_1(residual)
        x += residual
        x = F.relu(x, inplace=True)

        return x

# implementation 2
class ResidualUnitB(nn.Module):
    def __init__(self):
        super(ResidualUnitB, self).__init__()
        self.conv_0 = nn.Conv2d(3, 24, 3, padding=1, bias=False)
        self.bn_0 = nn.BatchNorm2d(24)
        self.conv_1 = nn.Conv2d(24, 24, 3, padding=1, bias=False)
        self.bn_1 = nn.BatchNorm2d(24)

    def forward(self, x):
        x = self.conv_0(x)
        x = self.bn_0(x)
        x = F.relu(x, inplace=True)

        identity = x
        x = self.conv_1(x)
        x = self.bn_1(x)
        x += identity
        x = F.relu(x, inplace=True)

        return x

I find that ResidualUnitA will raise the RuntimeError, but ResidualUnitB won’t. How to explain this using the restrictions of in-place operations?