What I wanted to do is set the top K values to 1’s and the rest to zero. Here I use topk() and scatter() to do the work. Take that resulting variable.data and set that to the l2.data (and l3).  This keeps the graph in tact since l2 and l3 store graph and grad data  for subsequent backprop.
x = Variable(torch.randn(N, shape[0]).type(dtype), requires_grad=False)
y = Variable(torch.randn(N, shape[2]).type(dtype), requires_grad=False)
k = [25, 50]

learning_rate = 0.001
for t in range(epochs):
    l2 = x.mm(w1)
    #these three lines solve the problem
    topk, indices = torch.topk(l2, k[1])
    topk.data = torch.ones(l2.size)
    l2.data = Variable(torch.zeros(l2.size)).scatter(1, indices, topk).data

    l3 = l2.mm(w2)
    #same here
    topk, indices = torch.topk(l3, k[1])
    topk.data = torch.ones(l3.size)
    l3.data = Variable(torch.zeros(l3.size)).scatter(1, indices, topk).data

    loss = (l3_sp - y).abs().sum()
    loss.backward()

    w1.data -= learning_rate * w1.grad.data
    w2.data -= learning_rate * w2.grad.data

    w1.grad.data.zero_()
    w2.grad.data.zero_()