When I tried to modify a CNN visualization method named gradient CAM to work for my YOLO v3 model, I met a problem that the gradient calculated contains all zeros which cannot be used for the following process after calling score.backward(). The score is the bounding box’s confidence regarding to that class and this value has been processed by NMS.
Any hint or help would be appreciated!
class GradCAM(object):
    """
    1: the network does not update gradient, input requires the update
    2: use targeted class's score to do backward propagation
    """

    def __init__(self, net, layer_name):
        self.net = net
        self.layer_name = layer_name
        self.feature = None
        self.gradient = None
        self.net.eval()
        self.handlers = []
        self._register_hook()

    def _get_features_hook(self, module, input, output):
        self.feature = output
        print("feature shape:{}".format(output.size()))

    def _get_grads_hook(self, module, input_grad, output_grad):
        """
        :param input_grad: tuple, input_grad[0]: None
                                   input_grad[1]: weight
                                   input_grad[2]: bias
        :param output_grad:tuple,长度为1
        :return:
        """
        #print(input_grad)
        #print(output_grad)
        self.gradient = output_grad[0]

    def _register_hook(self):
        for i, module in enumerate(self.net.module_list):
            if module == self.layer_name:
                self.handlers.append(module.register_forward_hook(self._get_features_hook))
                self.handlers.append(module.register_backward_hook(self._get_grads_hook))

    def remove_handlers(self):
        for handle in self.handlers:
            handle.remove()

    def __call__(self, inputs, index=0):
        """
        :param inputs: {"image": [C,H,W], "height": height, "width": width}
        :param index: 第几个边框
        :return:
        """
        self.net.zero_grad()
        output = self.net(inputs['image'])[0]
        output_nonmax = utils.non_max_suppression(output, conf_thres=0.25, iou_thres=0.45, multi_label=True)[0]
        print(output_nonmax)
        score = output_nonmax[index][4]
        print(score)
        score.retain_grad()
        #score = output[0]['instances'].scores[index]
        #proposal_idx = output[0]['instances'].indices[index]  
        proposal_idx = index
        score.backward()
        print(score.is_leaf)
        
        print(score.grad)
        gradient = self.gradient[proposal_idx].cpu().data.numpy()  # [C,H,W]
        #print(gradient)
        weight = np.mean(gradient, axis=(1, 2))  # [C]
        #print(weight)
        feature = self.feature[proposal_idx].cpu().data.numpy()  # [C,H,W]

        cam = feature * weight[:, np.newaxis, np.newaxis]  # [C,H,W]
        cam = np.sum(cam, axis=0)  # [H,W]
        cam = np.maximum(cam, 0)  # ReLU
        # normalize
        cam = cam - np.min(cam)
        cam = cam / np.max(cam)
        
        # resize to 224*224
        box = output[index][:4].detach().numpy().astype(np.int32)
        x1, y1, x2, y2 = box
        cam = cv2.resize(cam, (x2 - x1, y2 - y1))

        class_id = output[index][-1].detach().numpy()
        return cam, box, class_id