Thanks for the modified code. I am getting different results than you when I run a slightly modified version of your code. My results suggest that ReLU being in-place does make a significant difference, even for custom modules.
Here is the modified code that I ran:
import torch
from torch import nn

class MyConv(nn.Module):
    def __init__(self):
        super().__init__()
        self.weight = nn.parameter.Parameter(torch.randn(1, 1, 64, 64))
        

    def forward(self, x):
        return (x * self.weight)


def evaluate_memory(module):
    x = torch.rand(32, 32, 64, 64).cuda()
    torch.cuda.reset_peak_memory_stats()
    y = module(x)
    return torch.cuda.max_memory_allocated(torch.device("cuda")) / (1024 ** 2)

net_1 = nn.Sequential()
net_2 = nn.Sequential()

use_default_conv = True

for i in range(50):
    if use_default_conv:
        net_1.add_module("conv_{}".format(i), nn.Conv2d(32, 32, 3, padding=1))
        net_2.add_module("conv_{}".format(i), nn.Conv2d(32, 32, 3, padding=1))
    else:
        net_1.add_module("conv_{}".format(i), MyConv())
        net_2.add_module("conv_{}".format(i), MyConv())
    net_1.add_module("relu_{}".format(i), nn.ReLU(True))
    net_2.add_module("relu_{}".format(i), nn.ReLU(False))

net_1.cuda()
net_2.cuda()

m_1 = evaluate_memory(net_1)
m_2 = evaluate_memory(net_2)
print("memory for in-place mode: {:.1f}".format(m_1))
print("memory for out-place mode: {:.1f}".format(m_2))


When I run this with use_default_conv = True I get:
memory for in-place mode: 819.6
memory for out-place mode: 1619.6
When I run this with use_default_conv = False I get:
memory for in-place mode: 817.6
memory for out-place mode: 1617.6
This tells that ReLU being in-place does make the same degree of difference, in both cases.