You use index_select 4 times in your code:

  
      github.com
  
  
    pcshih/pytorch-VSLUD/blob/b3ed7aba9332d2e0a21a66e84eae3654c9e254af/SK.py#L38-L40 1

index = torch.tensor(column_mask, device=torch.device('cuda:0'))
h_select = torch.index_select(h, 3, index)
x_select = torch.index_select(x_temp, 3, index)



  
  
    
    
  
  



  
      github.com
  
  
    pcshih/pytorch-VSLUD/blob/b3ed7aba9332d2e0a21a66e84eae3654c9e254af/training_set_preparation.py#L42-L47

gt_summary = torch.from_numpy(dataset[key]["gtsummary"][...]).to(device)
column_index = gt_summary.nonzero().view(-1)
feature_summary_cuda = torch.from_numpy(dataset[key]["features"][...]).to(device)
feature_summary_cuda = feature_summary_cuda.transpose(1,0).view(1,1024,1,feature_summary_cuda.shape[0])
feature_summary_cuda = torch.index_select(feature_summary_cuda, 3, column_index)
attributes["summary_features"] = feature_summary_cuda; #print(torch.isnan(feature_summary_cuda).nonzero().view(-1))



  
  
    
    
  
  



  
      github.com
  
  
    pcshih/pytorch-VSLUD/blob/b3ed7aba9332d2e0a21a66e84eae3654c9e254af/train.py#L195-L198

index = torch.tensor(column_mask, device=device)
select_vd = torch.index_select(vd, 3, index)
reconstruct_loss = torch.norm(S_K_summary-select_vd, p=2)**2
reconstruct_loss /= len(column_mask)



  
  
    
    
  
  


Where is the problem?