@gphilip
@ptrblck
This is the executable code:
import torch
from torch import nn

class conv(torch.autograd.Function):
    @staticmethod
    def forward(ctx, x):
        y = 2 * x
        ctx.save_for_backward(y)
        return torch.sin(y)
    
    @staticmethod
    def backward(ctx, grad_out):
        y = ctx.saved_tensors
        with torch.no_grad():
            if ctx.needs_input_grad[0]:
                return grad_out * torch.cos(y)
            else:
                return None
    

class MyConv(nn.Module):
    def __init__(self):
        super().__init__()
        self.weight = nn.parameter.Parameter(torch.randn(1, 1, 64, 64))
        self.main = conv.apply

    def forward(self, x):
        return self.main(x) * self.weight


def evaluate_memory(module):
    x = torch.randn(32, 32, 64, 64).cuda()
    torch.cuda.reset_peak_memory_stats()
    y = module(x)
    return torch.cuda.max_memory_allocated() / (1024 ** 2)

net_1 = nn.Sequential()
net_2 = nn.Sequential()

use_default_conv = False

for i in range(50):
    if use_default_conv:
        net_1.add_module("conv_{}".format(i), nn.Conv2d(32, 32, 3, padding=1))
        net_2.add_module("conv_{}".format(i), nn.Conv2d(32, 32, 3, padding=1))
    else:
        net_1.add_module("conv_{}".format(i), MyConv())
        net_2.add_module("conv_{}".format(i), MyConv())
    net_1.add_module("relu_{}".format(i), nn.ReLU(True))
    net_2.add_module("relu_{}".format(i), nn.ReLU(False))

net_1.cuda()
net_2.cuda()

m_1 = evaluate_memory(net_1)
m_2 = evaluate_memory(net_2)
print("memory for in-place mode: {:.1f}".format(m_1))
print("memory for out-place mode: {:.1f}".format(m_2))


when set use_defualt_conv=True, the output is
memory for in-place mode: 819.6
memory for out-place mode: 1619.6

when set use_defualt_conv=False, the output is
memory for in-place mode: 2401.6
memory for out-place mode: 2417.6

My point is that when set use_defualt_conv=False, the memory of out-place mode should be half of in-place mode. However, it is not.