Dear jpeg729,
Thanks for help:
def forward(self, q, d_pos, d_neg):
        """ apply rel
            q: LongTensor (BS, qlen) Variable input
            d: LongTensor (BS, dlen) Variable input
            returns R1, R2, R3: relevance of 3 level (BS,)
        """
        q_mask = torch.ne(q, 0).unsqueeze(2).float()  # (BS, qlen, 1)
        q_mask = q_mask.detach()
        d_pos_mask = torch.ne(d_pos, 0).unsqueeze(2).float()  # (BS, dlen, 1)
        d_pos_mask = d_pos_mask.detach()
        d_neg_mask = torch.ne(d_neg, 0).unsqueeze(2).float()  # (BSm dlen, 1)
        d_neg_mask = d_neg_mask.detach()
        q_emb = self.emb_mod(q) * q_mask  # (BS, qlen, emb_size)
        d_pos_emb = self.emb_mod(d_pos) * d_pos_mask  # (BS, dlen, emb_size)
        d_neg_emb = self.emb_mod(d_neg) * d_neg_mask  # (BS, dlen, emb_size)
        # do convs
        q_conved1 = self.q_conv1(q_emb.transpose(1, 2))  # (BS, hs, qLen1)
        q_conved2 = self.q_conv2(q_conved1)  # (BS, hs, 1)
        d_pos_conved1 = self.d_conv1(d_pos_emb.transpose(1, 2))  # (BS, hs, dLen1)
        d_pos_conved2 = self.d_conv2(d_pos_conved1)  # (BS, hs, 1)
        d_neg_conved1 = self.d_conv1(d_neg_emb.transpose(1, 2))  # (BS, hs, dLen1)
        d_neg_conved2 = self.d_conv2(d_neg_conved1)  # (BS, hs, 1)
         # interactions matrices
        if self.sim_type == "Dot" or self.sim_type == "Cos":
            interact_pos_mat1 = self.sim(q_emb, d_pos_emb)  # (BS, qlen, dlen)
            interact_pos_mat2 = self.sim(q_conved1.transpose(1, 2),
                                         d_pos_conved1.transpose(1, 2))  # (BS, qLen1, dLen1)
            interact_neg_mat1 = self.sim(q_emb, d_neg_emb)  # (BS, qlen, dlen)
            interact_neg_mat2 = self.sim(q_conved1.transpose(1, 2),
                                         d_neg_conved1.transpose(1, 2))  # (BS, qLen1, dLen1)
       # ..... some ops on the interaction matrices
       #. to get scores S_pos, S_neg
       return S_pos, S_neg

Actually I do need these masks, because I randomly initialized the PAD embedding vector and since the PAD embedding vector will also be learnable, I must mask it to 0 vector and then do the COS interaction, otherwise there will be non 0 interaction signal cos(PAD, PAD) which will cause a lot of noises in my model.