i was trying to replace all the Relu activation functions with TLU of skresnet34 model(from timm library) as shown here : PyTorch-FilterResponseNormalizationLayer/frn.py at master · yukkyo/PyTorch-FilterResponseNormalizationLayer · GitHub 1 and in filter response normalization layer paper.
i was trying this :

class TLU(nn.Module):
    def __init__(self, num_features = 32):
        """max(y, tau) = max(y - tau, 0) + tau = ReLU(y - tau) + tau"""
        super(TLU, self).__init__()
        self.num_features = num_features
        self.tau = nn.parameter.Parameter(
            torch.Tensor(1, num_features, 1, 1), requires_grad=True)
        self.reset_parameters()

    def reset_parameters(self):
        nn.init.zeros_(self.tau)

    def extra_repr(self):
        return 'num_features={num_features}'.format(**self.__dict__)

    def forward(self, x):
        return torch.max(x, self.tau)
    

def convert_relu_to_tlu(model):
    for child_name, child in model.named_children():
        if isinstance(child, nn.ReLU):
            setattr(model, child_name, TLU())
        else:
            convert_relu_to_tlu(child)

def getter(model, name):
    layer = model
    for attrib in name.split("."):
        layer = getattr(layer, attrib)
    return layer

def setter(model, name, layer):
    try:
        attrib, name = name.rsplit(".", 1)
        model = getter(model, attrib)
    except ValueError:
        pass
    setattr(model, name, layer)

for name, module in model.named_modules():
    if isinstance(module, nn.ReLU):
        relu = getter(model, name)
        print(relu)
        #tlu = TLU(num_features = relu.num_features)
        tlu = TLU()
        print("changing {} with {}".format(relu, tlu))

        setter(model, name, tlu)

it replaces all relu with TLU but i can’t pass exact value of num_features in TLU while replacing relu with tlu,how to modify this line of code :

tlu = TLU(num_features = relu.num_features)

for changing all relu with tlu?