I am trying to implement this paper https://arxiv.org/pdf/1801.09403.pdf 27
The idea is to define a custom activation function using basic functions like relu, tanh etc with affine or convex constraints.
Here is a class structure I created but can’t make it work
 class combine_act(nn.Module):
      def __init__(self, in_features,act,comb='affine'):
           super(combine_act,self).__init__()
  
           self.in_features = in_features
           self.act = act                       #list of activations to use
           self.comb = comb                #type of combination [affine or convex]
           self.N = len(self.act)      
  
           self.alpha = nn.Parameter(torch.randn(len(act)))
           self.alpha.requiresGrad = True
  
  
           Lin = lambda x: x
  
           activations = {'linear': Lin, 'relu': nn.ReLU(), 'tanh': nn.Tanh()}
  
  
           def my_softmax(self, x):
                 means = torch.mean(x, 1, keepdim=True)[0]
                 x_exp = torch.exp(x-means)
                 x_exp_sum = torch.sum(x_exp, 1, keepdim=True)
                 return x_exp/x_exp_sum
  

           
  
           def get_alpha(self):
               self.eps = 1e-7
               if comb=='affine':
               return self.alpha / self.alpha.sum(1, keepdim=True).clamp(min=self.eps)
           else:
               return self.my_softmax(self.alpha)     

    
           def forward(self, x):
               out=x-x 
               weight = self.get_alpha()
               for i in range(self.N):
                    activation = activations[self.act[i]]
                    out += weight[i]*activation(x)
               return out

Call to this function would look like this:
x = nn.Linear(100,100)(x)
x = combine_act([‘linear’, ‘relu’, ‘tanh’],comb=‘affine’)(x)

I am getting this error:   ‘combine_act’ object has no attribute ‘dim’
Thanks in advance?