We want to realize a net that sent the vgg features into the lstm, then generate a tensor (2,3) and a fc layer for classification. For the generated (2,3) tensor, we want some elements to be 0. So we use the tensor arr to element-wise multiplication with m1. And then the m1 continue to be sent to the next .
What’s wrong with me in the process of forward？
How to realize the element-wise multiplication in the forward?
Another question ,I want to make my own loss function that need to use the element of m1,m2,m3. How can i get the element from the tensor to make my loss function.
class ST_LSTM(nn.Module):
    def __init__(self,features,num_classes):
        super(ST_LSTM,self).__init__()
        self.features = features
        for p in self.features.parameters():
            p.requires_grad=False

        self.classifier2 = nn.Sequential(
            nn.Linear(1024, 1024),
            nn.ReLU(True),
            nn.Dropout(),
            nn.Linear(1024, 30),
        ) 
        self.mm = nn.Sequential(
            nn.Linear(1024, 512),
            nn.ReLU(True),
            nn.Dropout(),
            nn.Linear(512, 6),
        )
        self.lstm = nn.LSTM(7*7*512,1024,batch_first=True)

# stn is the Spatial Transformer Networks from the pytorch Tutorial
    def stn(self, x, theta):
        # x = self.features(x)
        # xs = self.localization(x)
        # xs = xs.view(-1, 10 * 3 * 3)
        # theta = self.fc_loc(xs)
        # theta = theta.view(-1, 2, 3)
        grid = F.affine_grid(theta, x.size())
        x = F.grid_sample(x, grid)
        return x

    def forward(self, x):
        y =self.features(x)
        # z1 = self.stn(x) 
        z1= y.view(-1,1,7*7*512)
        r1, (h1, c1) = self.lstm(z1)
        s1 = self.classifier2(r1[:,-1,:]) 
        m1 = self.mm(r1[:,-1,:])
        
        m1 = m1.view(-1,2,3)
        arr = torch.cuda.FloatTensor([1, 0, 1, 0, 1, 1])
        # arr = np.array([1,0,1,0,1,1])
        # arr = torch.from_numpy(arr)
       # 16 is batchsize we want to make a tensor(batchsize,2,3) to element-wise mul with m1
        arr = arr.repeat(16,1)
        arr = arr.view(16,2,3)
        m1 = torch.mul(m1,arr)
        z2 = self.stn(y,m1)
        z2 = z2.view(-1,1,7*7*512)
        r2, (h2, c2) = self.lstm(z2,(h1,c1))
        s2 = self.classifier2(r2[:,-1,:]) 
        m2 = self.mm(r2[:,-1,:])

        m2 = m2.view(-1,2,3)
        z3 = self.stn(y,m2)
        z3 = z3.view(-1,1,7*7*512)
        r3, (h3, c3) = self.lstm(z3,(h2,c2))
        s3 = self.classifier2(r3[:,-1,:]) 
        return s1,s2,s3


image.png863×303 8.6 KB