Sorry, I think I missed some specific code information.
here is my complete code

import torch
from wsddn.roi_pooling.modules.roi_pool import RoIPool
from wsddn.utils.network import FC
from wsddn.utils import network
import torch.nn.functional as F
import torch.nn as nn
from wsddn.vgg16 import VGG16


class WSDDN(nn.Module):
    feature_scale = 1.0 / 16
    n_classes = 21

    def __init__(self, classes=None):
        super(WSDDN, self).__init__()
        if classes is not None:
            self.classes = classes
            self.n_classes = len(classes)

        self.features = VGG16()
        self.roi_pool = RoIPool(7, 7, self.feature_scale)
        self.fc6 = FC(512 * 7 * 7, 4096)
        self.fc7 = FC(4096, 4096)
        self.classifier_head = FC(4096, self.n_classes, relu=False)
        self.detection_head = FC(4096, self.n_classes, relu=False)

        self._loss = None
        self._detection = None

    def forward(self, im_data, rois, labels):
        im_data = network.np_to_variable(im_data, is_cuda=True)
        im_data = im_data.permute(0, 3, 1, 2)
        rois = network.np_to_variable(rois, is_cuda=True)
        labels = network.np_to_variable(labels, is_cuda=True)
        features = self.features(im_data)

        pooled_features = self.roi_pool(features, rois)
        x = pooled_features.view(pooled_features.size()[0], -1)
        x = self.fc6(x)
        x = F.dropout(x, training=self.training)
        x = self.fc7(x)
        x = F.dropout(x, training=self.training)
        cls_score = self.classifier_head(x)
        det_score = self.detection_head(x)

        cls_predict = F.softmax(cls_score, dim=1)
        det_predict = F.softmax(det_score, dim=0)
        predict = F.mul(cls_predict, det_predict)
        y_predict = predict.sum(dim=0)
        y_predict = y_predict[1:]

        self._loss = self.build_loss(y_predict, labels)
        self._detection = predict

        return y_predict

    @property
    def detection(self):
        return self._detection

    @property
    def loss(self):
        return self._loss

    def build_loss(self, y_predict, labels):
        loss = F.binary_cross_entropy(y_predict, labels, size_average=False)
        # y_predict = torch.clamp(y_predict, min=1e-4, max=1 - 1e-4)
        # loss = -1 * torch.log(labels * (y_predict - 1.0 / 2) + 1 / 2).sum()
        return loss











and the weird thing is, this problem will occur after training for about 10000 iters, so I’m waiting for the problem now