Thanks for your reply.
For example in the following case,   Do I need to add self.out.grad.data.fill_(0) to the  mode.zero_grad() function?
    def __init__(self):
        self.out = torch.autograd.Variable(torch.zeros(timestep, batchsize, 
                                                       self.W_decode.size()[1]) , requires_grad=True)

    def forward(self, input, state=None):
        # input is timesetep*N
        batchsize, timestep = input.size()[1], input.size()[0]
        vec_input = input.view(-1)
        emb = torch.index_select(self.W_emb, 0, vec_input).view(timestep,batchsize, -1) # emb = N*ninp
        inp = matmul(emb, self.W_rnn)
        state = torch.autograd.Variable( torch.zeros(inp.size()[1:])) if state is None else state
        for step in range(inp.size()[0]):
            this_input = inp[step]  # N * nhid
            this_input = torch.addmm(this_input, state, self.U_rnn)
            state = F.tanh(this_input  + self.b_rnn.expand_as(this_input) )
            self.out[step] = torch.addmm(self.out[step], state, self.W_decode)
            self.out[step] = F.softmax(self.out[step]  + self.b_decode.expand_as(out[step]) )
        return self.out