You could do something like
class Model(nn.Module):
    def init(self, args):
        self.m1 = Module1()
        if args.use_my_second_layer:
             self.m2 = Module2()
    def forward(self, x):
        h = self.m1(x)
        if args.use_my_second_layer:
             h = self.m2(x)
        return h