Thanks for the code.
It seems alpha gets valid gradients (they look pretty high, but I’m not sure, if that’s desired in your use case).
class combine_act(nn.Module):
    def __init__(self,in_features,comb='convex'):
        super(combine_act, self).__init__()
    
        self.in_features = in_features
        self.comb = comb                     #type of combination [affine or convex]
    
        Lin = lambda x: x

        self.activations = {Lin, nn.ReLU(), nn.Tanh()}           #list of activations to use 

        self.alpha = nn.Parameter(torch.randn(len(self.activations)))

    def my_softmax(self, x):
        means = torch.mean(x)
        x_exp = torch.exp(x-means)
        x_exp_sum = torch.sum(x_exp)
        return x_exp/x_exp_sum

    def unitnorm(self,x):
        eps = 1e-7
        return x / x.sum().clamp(min=eps)

    def get_alpha(self,a):
        if self.comb=='affine':
           return self.unitnorm(a).cuda()
        else:
           return self.my_softmax(a).cuda()

    def forward(self, x):
        out=x-x 
        weight = self.get_alpha(self.alpha)
        for i, activation in enumerate(self.activations):
           out += weight[i]*activation(x)
        return out


x = torch.randn(1, 100).cuda()    
model = combine_act(100, comb='affine').cuda()

x = model(x)
x.mean().backward()
print(model.alpha.grad)

You could simply pass alpha to an optimizer, which would perform the update step.
Could you clarify the use case a bit, if that wouldn’t work?