Thanks for the swift reply I will post the entire function that produces the problem, this will provide some more context.
def rgb_to_hsv(image):

    torch.autograd.set_detect_anomaly(True)
    r = image[..., 0, :, :]
    g = image[..., 1, :, :]
    b = image[..., 2, :, :]

    maxc = image.max(-3)[0]
    minc = image.min(-3)[0]

    v = maxc  # brightness

    deltac = maxc - minc
    s = deltac.clone() / v  # saturation

    deltac[deltac == 0] = 1  # avoid division by zero
    rc = (maxc - r) / deltac
    gc = (maxc - g) / deltac
    bc = (maxc - b) / deltac

    h = 4.0 + gc - rc
    h[g == maxc] = 2.0 + rc[g == maxc] - bc[g == maxc]
    h[r == maxc] = bc[r == maxc] - gc[r == maxc]
    h[minc == maxc] = 0.0

    h = (h / 6.0) % 1.0

    return torch.stack([h, s, v], dim=-3)


The code for checking the gradients:
from torch.autograd import gradcheck
 def test_gradcheck(self):

        data = torch.tensor([[[[21., 22.],
                              [22., 22.]],

                             [[13., 14.],
                              [14., 14.]],

                             [[8., 8.],
                              [8., 8.]]]])  # 3x2x2

        data = utils.tensor_to_gradcheck_var(data)  # to var

        assert gradcheck(image.RgbToHsV(), (data,),
                         raise_exception=True)