ptrblck:

Conv1d


Hi again ptrblck,
I tried to did what you suggest but it did not change anything I guess. Could you check ( if possible) this is what you really suggested ?
There are more than one convolution layer with different kernel sizes in my imlpementation but they all perform on the same embedding matrix. Therefore, If I can fix one of them, I would be able to apply the same thing to all.
class Anger2(nn.Module):
    # out_channel: # output channel of convolutions (100)
    # dense_out :  1st dense layer output  dimension (30)
    def __init__(self,embedding_matrix,out_channel,dense_out,n_gram_num):
        super(Anger2, self).__init__()
        num_embeddings,embedding_dim = embedding_matrix.shape
        self.embedding = nn.Embedding(num_embeddings,embedding_dim)
        self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix,dtype=torch.float))
        self.conv2  = nn.Conv1d(embedding_dim,out_channel, kernel_size = 2)
        self.conv3  = nn.Conv1d(embedding_dim,out_channel, kernel_size = 3)
        self.conv4  = nn.Conv1d(embedding_dim,out_channel, kernel_size = 4)
        self.conv5  = nn.Conv1d(embedding_dim,out_channel, kernel_size = 5)
        self.conv6  = nn.Conv1d(embedding_dim,out_channel, kernel_size = 6) 
        # https://pytorch.org/docs/stable/nn.html#torch.nn.BatchNorm1d
        self.dense1_bn = nn.BatchNorm1d(out_channel*n_gram_num)
        self.drop = nn.Dropout(p=0.5)
        self.fc1 = nn.Linear(out_channel*n_gram_num, dense_out)
        self.fc2 = nn.Linear(dense_out, 1)

     
    def forward(self,x,y):
        # x shape: BSxMAX_LEN
        embeds     = self.embedding(x) # BS(64) x MAX_LEN(80) x EMBED_SIZE(200)
        embeds = embeds.view(x.shape[0],-1,x.shape[1]) 

        o2 = self.conv2(embeds)
        o2 = torch.squeeze(F.max_pool1d(o2,o2.shape[2]))

        o3 = self.conv3(embeds)
        o3 = torch.squeeze(F.max_pool1d(o3,o3.shape[2]))
        
        o4 = self.conv4(embeds)
        o4 = torch.squeeze(F.max_pool1d(o4,o4.shape[2]))
        
        o5 = self.conv5(embeds)
        o5 = torch.squeeze(F.max_pool1d(o5,o5.shape[2]))
 
        o6 = self.conv6(embeds)
        o6 = torch.squeeze(F.max_pool1d(o6,o6.shape[2]))

        concatenated = torch.cat((o2,o3,o4,o5,o6), 1) # 64x500
        concatenated = self.dense1_bn(concatenated)
        out = self.drop(concatenated)
        out = F.sigmoid(self.fc1(out))
        out = F.sigmoid(self.fc2(out))
        return out