If you manully place some values of a tensor to zeros, it will not compute the gradients properly.
I implent a k sparse filter, you can test it out.
import torch


class KSparse(torch.nn.Module):
    def __init__(self, k):

        super(KSparse, self).__init__()
        self.k = k

    def forward(self, x):

        _, indices = torch.topk(x, self.k)
        mask = torch.zeros(x.size()).cuda()
        mask.scatter_(2, indices, 1)
        return torch.mul(x, mask)


test = torch.tensor([[[1., 2., 4.], [2., 3., 1.]]]).cuda()
k_sparse = KSparse(1)
print(k_sparse(test))