pascal_notsawo:

optimizer.zero_grad()


I still got same result but if i am not using gat layer and i change output to fully connected linear the loss value will decreased and model managed to learn. No matter optimizer.zero_grad() position, below or above scores the model still managed to learn.
I change the code like this
def forward(self, x, feat): # feat.size : (N, 768)

        features, _ = self.lstm(x)
        features = features[:, -1, :].squeeze(1) #features.size : (batch_size, hidden_size*2)
        
        out = self.fc(features)

        return out

the result
0 0.266863764077425
1 0.17352587655186652
2 0.1687601201236248
3 0.16377742290496827
4 0.15879809260368347

I try to track what paramers were not updated:
The code like this
# Train Network
old_params = {}
for name, params in model.named_parameters():
    old_params[name] = params.clone()

for epoch in range(num_epochs):

# perform update
    loss_list = []
    hammingloss = []
    microf1_score= []
    for start, end in indexloader(datalen, batch_size):
        data = X_train[start:end]
        data = data.to(device)
        targets = y_train[start:end]
        targets = targets.to(device)
        optimizer.zero_grad()
        # forward
        scores = model(data, feat)
        loss = criterion(scores, targets)
        # backward
        # optimizer.zero_grad()
        loss.backward()
        clip_grad_norm_(model.parameters(), max_norm=10)
        loss_list.append(loss.item())
        # gradient descent or adam step
        optimizer.step()
    print(epoch, np.mean(loss_list))
    for name, params in model.named_parameters():
      if (old_params[name] == params).all():
        print(name, ' true')
      else:
        print(name, ' false')

And when i am using GAT all parameters were not updated.
the result
A  true
lstm.weight_ih_l0  true
lstm.weight_hh_l0  true
lstm.bias_ih_l0  true
lstm.bias_hh_l0  true
lstm.weight_ih_l0_reverse  true
lstm.weight_hh_l0_reverse  true
lstm.bias_ih_l0_reverse  true
lstm.bias_hh_l0_reverse  true
gat1.attention_0.W  true
gat1.attention_1.W  true
gat1.attention_2.W  true
gat1.attention_3.W  true
gat2.attention_0.W  true
gat2.attention_1.W  true
gat2.attention_2.W  true
gat2.attention_3.W  true

but if i am not using GAT layer, lstm and fc layer weights will be updated:
the result:
A  true
lstm.weight_ih_l0  false
lstm.weight_hh_l0  false
lstm.bias_ih_l0  false
lstm.bias_hh_l0  false
lstm.weight_ih_l0_reverse  false
lstm.weight_hh_l0_reverse  true
lstm.bias_ih_l0_reverse  false
lstm.bias_hh_l0_reverse  false
gat1.attention_0.W  true
gat1.attention_1.W  true
gat1.attention_2.W  true
gat1.attention_3.W  true
gat2.attention_0.W  true
gat2.attention_1.W  true
gat2.attention_2.W  true
gat2.attention_3.W  true
fc.weight  false
fc.bias  false