Ah OK, I clearly misunderstood the use case.
This code should work as the suggested one by Adam 
class MyModel(nn.Module):
    def __init__(self):
        super(MyModel, self).__init__()
        self.submodule = None
        
    def forward(self, x):
        if self.submodule is None:
            self.submodule = nn.Sequential(
                nn.Linear(1, 1),
                nn.ReLU()
            )
        x = self.submodule(x)
        return x

model = MyModel()
model(torch.randn(1, 1))
print(dict(model.named_parameters()))
> {'submodule.0.weight': Parameter containing:
tensor([[-0.1282]], requires_grad=True), 'submodule.0.bias': Parameter containing:
tensor([-0.7143], requires_grad=True)}