My Model:
    
class myNet(nn.Module):
    def __init__(self):
        super(myNet,self).__init__()
        backbone=geffnet.efficientnet_b3(pretrained=True)
        act1=Dynamic_relu_b(40)
        act2=Dynamic_relu_b(1536)
      
        self.backbone=torch.nn.Sequential(
            backbone.conv_stem,
            backbone.bn1,
            act1,
            backbone.blocks,
            backbone.conv_head,
            backbone.bn2,
            act2,
            backbone.global_pool
        )
        
        self.global_avgpool=torch.nn.AdaptiveAvgPool2d(1)
        self.global_bn=nn.BatchNorm1d(1536)
        self.global_bn.bias.requires_grad=False
        self.local_conv=nn.Conv2d(1536,512,1)
        self.local_bn=nn.BatchNorm2d(512)
        self.local_bn.bias.requires_grad=False
        self.fc=nn.Linear(1536,20)
        nn.init.kaiming_normal_(self.fc.weight,mode='fan_out')
        nn.init.constant_(self.fc.bias,0)

    def forward(self,x):
        x=self.backbone(x)

        global_feat=self.global_avgpool(x)
        global_feat=global_feat.view(global_feat.shape[0],-1)
        global_feat=F.dropout(global_feat,p=0.2)
        global_feat=self.global_bn(global_feat)
        global_feat=l2_norm(global_feat)
    
        local_feat=torch.mean(x,-1,keepdim=True)
        local_feat=self.local_bn(self.local_conv(local_feat))
        local_feat=local_feat.squeeze(-1).permute(0,2,1)
        local_feat=l2_norm(local_feat,axis=-1)

        out=self.fc(global_feat)*16
        return global_feat,local_feat,out

def one_hot_smooth_label(x,num_class,smooth=0.1):
    num=x.shape[0]
    labels=torch.zeros((num,20))
    for i in range(num):
        labels[i][x[i]]=1
    labels=(1-(num_class-1)/num_class*smooth)*labels+smooth/num_class
    return labels

images=torch.rand((4,3,300,300))
images=images.cuda().float()
labels=torch.from_numpy(np.array([1,0,0,1]))
model=myNet()
model=model.cuda()
global_feat,local_feat,cls_score=model(images)
# global_feat=global_feat.to('cpu')
# local_feat=local_feat.to('cpu')
cls_score=cls_score.to('cpu')
labels=one_hot_smooth_label(labels,20)
criterion=nn.BCEWithLogitsLoss()

loss=criterion(cls_score,labels)
loss.backward()

This the error:
RuntimeError                              Traceback (most recent call last)
<ipython-input-39-951a8a08a40d> in <module>
     20 
     21 loss=criterion(cls_score,labels)
---> 22 loss.backward()

/opt/conda/lib/python3.7/site-packages/torch/tensor.py in backward(self, gradient, retain_graph, create_graph)
    193                 products. Defaults to ``False``.
    194         """
--> 195         torch.autograd.backward(self, gradient, retain_graph, create_graph)
    196 
    197     def register_hook(self, hook):

/opt/conda/lib/python3.7/site-packages/torch/autograd/__init__.py in backward(tensors, grad_tensors, retain_graph, create_graph, grad_variables)
     97     Variable._execution_engine.run_backward(
     98         tensors, grad_tensors, retain_graph, create_graph,
---> 99         allow_unreachable=True)  # allow_unreachable flag
    100 
    101 

RuntimeError: Function AddBackward0 returned an invalid gradient at index 1 - expected type torch.FloatTensor but got torch.cuda.FloatTensor

After many experiments I found the problem：
#if we delete act1 and act2 in model,the error will not happen

class myNet(nn.Module):
    def __init__(self):
        super(myNet,self).__init__()
        backbone=geffnet.efficientnet_b3(pretrained=True)
        act1=Dynamic_relu_b(40)
        act2=Dynamic_relu_b(1536)
      
        self.backbone=torch.nn.Sequential(
            backbone.conv_stem,
            backbone.bn1,
            #act1,
            backbone.blocks,
            backbone.conv_head,
            backbone.bn2,
            #act2,
            backbone.global_pool
        )
        
        self.global_avgpool=torch.nn.AdaptiveAvgPool2d(1)
        self.global_bn=nn.BatchNorm1d(1536)
        self.global_bn.bias.requires_grad=False
        self.local_conv=nn.Conv2d(1536,512,1)
        self.local_bn=nn.BatchNorm2d(512)
        self.local_bn.bias.requires_grad=False
        self.fc=nn.Linear(1536,20)
        nn.init.kaiming_normal_(self.fc.weight,mode='fan_out')
        nn.init.constant_(self.fc.bias,0)

    def forward(self,x):
        x=self.backbone(x)

        global_feat=self.global_avgpool(x)
        global_feat=global_feat.view(global_feat.shape[0],-1)
        global_feat=F.dropout(global_feat,p=0.2)
        global_feat=self.global_bn(global_feat)
        global_feat=l2_norm(global_feat)
    
        local_feat=torch.mean(x,-1,keepdim=True)
        local_feat=self.local_bn(self.local_conv(local_feat))
        local_feat=local_feat.squeeze(-1).permute(0,2,1)
        local_feat=l2_norm(local_feat,axis=-1)

        out=self.fc(global_feat)*16
        return global_feat,local_feat,out


This is the Dynamic_relu code:

class Residual(nn.Module):
    def __init__(self, in_channel, R=8, k=2):
        super(Residual, self).__init__()
        self.avg = nn.AdaptiveAvgPool2d((1, 1))
        self.relu = nn.ReLU(inplace=True)
        self.R = R
        self.k = k
        out_channel = int(in_channel / R)
        self.fc1 = nn.Linear(in_channel, out_channel)
        fc_list = []
        for i in range(k):
            fc_list.append(nn.Linear(out_channel, 2 * in_channel))
        self.fc2 = nn.ModuleList(fc_list)

    def forward(self, x):
        x = self.avg(x)
        x = torch.squeeze(x)
        x = self.fc1(x)
        x = self.relu(x)
        result_list = []
        for i in range(self.k):
            result = self.fc2[i](x)
            result = 2 * torch.sigmoid(result) - 1
            result_list.append(result)
        return result_list


class Dynamic_relu_b(nn.Module):
    def __init__(self, inchannel, R=8, k=2):
        super(Dynamic_relu_b, self).__init__()
        self.lambda_alpha = 1
        self.lambda_beta = 0.5
        self.R = R
        self.k = k
        self.init_alpha = torch.zeros(self.k)
        self.init_beta = torch.zeros(self.k)
        self.init_alpha[0] = 1
        self.init_beta[0] = 1
        for i in range(1, k):
            self.init_alpha[i] = 0
            self.init_beta[i] = 0

        self.residual = Residual(inchannel)

    def forward(self, input):
        delta = self.residual(input)
        in_channel = input.shape[1]
        bs = input.shape[0]
        alpha = torch.zeros((self.k, bs, in_channel))
        beta = torch.zeros((self.k, bs, in_channel))
        for i in range(self.k):
            for j, c in enumerate(range(0, in_channel * 2, 2)):
                alpha[i, :, j] = delta[i][:, c]
                beta[i, :, j] = delta[i][:, c + 1]
        alpha1 = alpha[0]
        beta1 = beta[0]
        max_result = self.dynamic_function(alpha1, beta1, input, 0)
        for i in range(1, self.k):
            alphai = alpha[i]
            betai = beta[i]
            result = self.dynamic_function(alphai, betai, input, i)
            max_result = torch.max(max_result, result)
        return max_result

    def dynamic_function(self, alpha, beta, x, k):
        init_alpha = self.init_alpha[k]
        init_beta = self.init_beta[k]
        alpha = init_alpha + self.lambda_alpha * alpha
        beta = init_beta + self.lambda_beta * beta
        bs = x.shape[0]
        channel = x.shape[1]
        results = torch.zeros_like(x)
        for i in range(bs):
            for c in range(channel):
                results[i, c, :, :] = x[i, c] * alpha[i, c] + beta[i, c]
        return results

How can I solve this problem？