One way would be to register a hook and multiply it with a mask to zero out the unwanted gradients:
def zero_w1(mask):
    def hook(grad):
        grad = grad * mask
        return grad
    return hook

w1 = 0.1
W1 = nn.Parameter(torch.tensor([[[[w1,0],[0,0]]]]).expand(-1, 31, -1, -1))

mask = (W1 != 0.).float()
W1.register_hook(zero_w1(mask))
output = F.conv2d(x, W1, stride=2)
output.mean().backward()
print(W1.grad)