Ok my work around is to have a soft decision, not a hard decision. So do this:
class ChannelDecision(nn.Module):
    def __init__(self, inc):
        super(ChannelDecision, self).__init__()
        self.net    = nn.Sequential(nn.Conv2d(inc, inc, kernel_size=1, bias=True),
                                    nn.ReLU(inplace=True),
                                    nn.Conv2d(inc, inc, kernel_size=1, bias=True),
                                    nn.Softmax(dim=1))

    def forward(self, x):
        probs = self.net(x)
        x = x * probs
        x = x.sum(1, keepdim=True)
        return x