This is the code i run:
    repnet = RepNet()
    p = torch.zeros((5,64))
    l = torch.zeros((5,64))
    for i in range(2):
        dummy = torch.randn((5, 64, 3, 112, 112))
        repnet.optimizer.zero_grad()
        l_logits, p_logits = repnet(dummy)
        loss = torch.nn.BCELoss()(torch.nn.Sigmoid()(p_logits.view(-1)), p.float().view(-1))
        loss += torch.nn.CrossEntropyLoss()(l_logits.view(dummy.shape[0] * 64, 32), l.view(-1).long())
        loss.backward()
        clip_gradient(repnet.optimizer, 0.1)
        repnet.optimizer.step()
        for name, param in repnet.named_parameters():
            print(name, torch.isfinite(param.grad).all())

    

This is the output:
encoder.temporal_context.0.weight tensor(False)
encoder.temporal_context.0.bias tensor(False)
period_predictor.cnn.0.weight tensor(True)
period_predictor.cnn.0.bias tensor(True)
period_predictor.projection.weight tensor(True)
period_predictor.projection.bias tensor(True)
period_predictor.transformer.encoder.layers.0.self_attn.in_proj_weight tensor(True)
period_predictor.transformer.encoder.layers.0.self_attn.in_proj_bias tensor(True)
period_predictor.transformer.encoder.layers.0.self_attn.out_proj.weight tensor(True)
period_predictor.transformer.encoder.layers.0.self_attn.out_proj.bias tensor(True)
period_predictor.transformer.encoder.layers.0.linear1.weight tensor(True)
period_predictor.transformer.encoder.layers.0.linear1.bias tensor(True)
period_predictor.transformer.encoder.layers.0.linear2.weight tensor(True)
period_predictor.transformer.encoder.layers.0.linear2.bias tensor(True)
period_predictor.transformer.encoder.layers.0.norm1.weight tensor(True)
period_predictor.transformer.encoder.layers.0.norm1.bias tensor(True)
period_predictor.transformer.encoder.layers.0.norm2.weight tensor(True)
period_predictor.transformer.encoder.layers.0.norm2.bias tensor(True)
period_predictor.transformer.encoder.layers.1.self_attn.in_proj_weight tensor(True)
period_predictor.transformer.encoder.layers.1.self_attn.in_proj_bias tensor(True)
period_predictor.transformer.encoder.layers.1.self_attn.out_proj.weight tensor(True)
period_predictor.transformer.encoder.layers.1.self_attn.out_proj.bias tensor(True)
period_predictor.transformer.encoder.layers.1.linear1.weight tensor(True)
period_predictor.transformer.encoder.layers.1.linear1.bias tensor(True)
period_predictor.transformer.encoder.layers.1.linear2.weight tensor(True)
period_predictor.transformer.encoder.layers.1.linear2.bias tensor(True)
period_predictor.transformer.encoder.layers.1.norm1.weight tensor(True)
period_predictor.transformer.encoder.layers.1.norm1.bias tensor(True)
period_predictor.transformer.encoder.layers.1.norm2.weight tensor(True)
period_predictor.transformer.encoder.layers.1.norm2.bias tensor(True)
period_predictor.transformer.encoder.layers.2.self_attn.in_proj_weight tensor(True)
period_predictor.transformer.encoder.layers.2.self_attn.in_proj_bias tensor(True)
period_predictor.transformer.encoder.layers.2.self_attn.out_proj.weight tensor(True)
period_predictor.transformer.encoder.layers.2.self_attn.out_proj.bias tensor(True)
period_predictor.transformer.encoder.layers.2.linear1.weight tensor(True)
period_predictor.transformer.encoder.layers.2.linear1.bias tensor(True)
period_predictor.transformer.encoder.layers.2.linear2.weight tensor(True)
period_predictor.transformer.encoder.layers.2.linear2.bias tensor(True)
period_predictor.transformer.encoder.layers.2.norm1.weight tensor(True)
period_predictor.transformer.encoder.layers.2.norm1.bias tensor(True)
period_predictor.transformer.encoder.layers.2.norm2.weight tensor(True)
period_predictor.transformer.encoder.layers.2.norm2.bias tensor(True)
period_predictor.transformer.encoder.layers.3.self_attn.in_proj_weight tensor(True)
period_predictor.transformer.encoder.layers.3.self_attn.in_proj_bias tensor(True)
period_predictor.transformer.encoder.layers.3.self_attn.out_proj.weight tensor(True)
period_predictor.transformer.encoder.layers.3.self_attn.out_proj.bias tensor(True)
period_predictor.transformer.encoder.layers.3.linear1.weight tensor(True)
period_predictor.transformer.encoder.layers.3.linear1.bias tensor(True)
period_predictor.transformer.encoder.layers.3.linear2.weight tensor(True)
period_predictor.transformer.encoder.layers.3.linear2.bias tensor(True)
period_predictor.transformer.encoder.layers.3.norm1.weight tensor(True)
period_predictor.transformer.encoder.layers.3.norm1.bias tensor(True)
period_predictor.transformer.encoder.layers.3.norm2.weight tensor(True)
period_predictor.transformer.encoder.layers.3.norm2.bias tensor(True)
period_predictor.transformer.encoder.layers.4.self_attn.in_proj_weight tensor(True)
period_predictor.transformer.encoder.layers.4.self_attn.in_proj_bias tensor(True)
period_predictor.transformer.encoder.layers.4.self_attn.out_proj.weight tensor(True)
period_predictor.transformer.encoder.layers.4.self_attn.out_proj.bias tensor(True)
period_predictor.transformer.encoder.layers.4.linear1.weight tensor(True)
period_predictor.transformer.encoder.layers.4.linear1.bias tensor(True)
period_predictor.transformer.encoder.layers.4.linear2.weight tensor(True)
period_predictor.transformer.encoder.layers.4.linear2.bias tensor(True)
period_predictor.transformer.encoder.layers.4.norm1.weight tensor(True)
period_predictor.transformer.encoder.layers.4.norm1.bias tensor(True)
period_predictor.transformer.encoder.layers.4.norm2.weight tensor(True)
period_predictor.transformer.encoder.layers.4.norm2.bias tensor(True)
period_predictor.transformer.encoder.layers.5.self_attn.in_proj_weight tensor(True)
period_predictor.transformer.encoder.layers.5.self_attn.in_proj_bias tensor(True)
period_predictor.transformer.encoder.layers.5.self_attn.out_proj.weight tensor(True)
period_predictor.transformer.encoder.layers.5.self_attn.out_proj.bias tensor(True)
period_predictor.transformer.encoder.layers.5.linear1.weight tensor(True)
period_predictor.transformer.encoder.layers.5.linear1.bias tensor(True)
period_predictor.transformer.encoder.layers.5.linear2.weight tensor(True)
period_predictor.transformer.encoder.layers.5.linear2.bias tensor(True)
period_predictor.transformer.encoder.layers.5.norm1.weight tensor(True)
period_predictor.transformer.encoder.layers.5.norm1.bias tensor(True)
period_predictor.transformer.encoder.layers.5.norm2.weight tensor(True)
period_predictor.transformer.encoder.layers.5.norm2.bias tensor(True)
period_predictor.transformer.encoder.norm.weight tensor(True)
period_predictor.transformer.encoder.norm.bias tensor(True)
period_predictor.transformer.decoder.layers.0.self_attn.in_proj_weight tensor(True)
period_predictor.transformer.decoder.layers.0.self_attn.in_proj_bias tensor(True)
period_predictor.transformer.decoder.layers.0.self_attn.out_proj.weight tensor(True)
period_predictor.transformer.decoder.layers.0.self_attn.out_proj.bias tensor(True)
period_predictor.transformer.decoder.layers.0.multihead_attn.in_proj_weight tensor(True)
period_predictor.transformer.decoder.layers.0.multihead_attn.in_proj_bias tensor(True)
period_predictor.transformer.decoder.layers.0.multihead_attn.out_proj.weight tensor(True)
period_predictor.transformer.decoder.layers.0.multihead_attn.out_proj.bias tensor(True)
period_predictor.transformer.decoder.layers.0.linear1.weight tensor(True)
period_predictor.transformer.decoder.layers.0.linear1.bias tensor(True)
period_predictor.transformer.decoder.layers.0.linear2.weight tensor(True)
period_predictor.transformer.decoder.layers.0.linear2.bias tensor(True)
period_predictor.transformer.decoder.layers.0.norm1.weight tensor(True)
period_predictor.transformer.decoder.layers.0.norm1.bias tensor(True)
period_predictor.transformer.decoder.layers.0.norm2.weight tensor(True)
period_predictor.transformer.decoder.layers.0.norm2.bias tensor(True)
period_predictor.transformer.decoder.layers.0.norm3.weight tensor(True)
period_predictor.transformer.decoder.layers.0.norm3.bias tensor(True)
period_predictor.transformer.decoder.layers.1.self_attn.in_proj_weight tensor(True)
period_predictor.transformer.decoder.layers.1.self_attn.in_proj_bias tensor(True)
period_predictor.transformer.decoder.layers.1.self_attn.out_proj.weight tensor(True)
period_predictor.transformer.decoder.layers.1.self_attn.out_proj.bias tensor(True)
period_predictor.transformer.decoder.layers.1.multihead_attn.in_proj_weight tensor(True)
period_predictor.transformer.decoder.layers.1.multihead_attn.in_proj_bias tensor(True)
period_predictor.transformer.decoder.layers.1.multihead_attn.out_proj.weight tensor(True)
period_predictor.transformer.decoder.layers.1.multihead_attn.out_proj.bias tensor(True)
period_predictor.transformer.decoder.layers.1.linear1.weight tensor(True)
period_predictor.transformer.decoder.layers.1.linear1.bias tensor(True)
period_predictor.transformer.decoder.layers.1.linear2.weight tensor(True)
period_predictor.transformer.decoder.layers.1.linear2.bias tensor(True)
period_predictor.transformer.decoder.layers.1.norm1.weight tensor(True)
period_predictor.transformer.decoder.layers.1.norm1.bias tensor(True)
period_predictor.transformer.decoder.layers.1.norm2.weight tensor(True)
period_predictor.transformer.decoder.layers.1.norm2.bias tensor(True)
period_predictor.transformer.decoder.layers.1.norm3.weight tensor(True)
period_predictor.transformer.decoder.layers.1.norm3.bias tensor(True)
period_predictor.transformer.decoder.layers.2.self_attn.in_proj_weight tensor(True)
period_predictor.transformer.decoder.layers.2.self_attn.in_proj_bias tensor(True)
period_predictor.transformer.decoder.layers.2.self_attn.out_proj.weight tensor(True)
period_predictor.transformer.decoder.layers.2.self_attn.out_proj.bias tensor(True)
period_predictor.transformer.decoder.layers.2.multihead_attn.in_proj_weight tensor(True)
period_predictor.transformer.decoder.layers.2.multihead_attn.in_proj_bias tensor(True)
period_predictor.transformer.decoder.layers.2.multihead_attn.out_proj.weight tensor(True)
period_predictor.transformer.decoder.layers.2.multihead_attn.out_proj.bias tensor(True)
period_predictor.transformer.decoder.layers.2.linear1.weight tensor(True)
period_predictor.transformer.decoder.layers.2.linear1.bias tensor(True)
period_predictor.transformer.decoder.layers.2.linear2.weight tensor(True)
period_predictor.transformer.decoder.layers.2.linear2.bias tensor(True)
period_predictor.transformer.decoder.layers.2.norm1.weight tensor(True)
period_predictor.transformer.decoder.layers.2.norm1.bias tensor(True)
period_predictor.transformer.decoder.layers.2.norm2.weight tensor(True)
period_predictor.transformer.decoder.layers.2.norm2.bias tensor(True)
period_predictor.transformer.decoder.layers.2.norm3.weight tensor(True)
period_predictor.transformer.decoder.layers.2.norm3.bias tensor(True)
period_predictor.transformer.decoder.layers.3.self_attn.in_proj_weight tensor(True)
period_predictor.transformer.decoder.layers.3.self_attn.in_proj_bias tensor(True)
period_predictor.transformer.decoder.layers.3.self_attn.out_proj.weight tensor(True)
period_predictor.transformer.decoder.layers.3.self_attn.out_proj.bias tensor(True)
period_predictor.transformer.decoder.layers.3.multihead_attn.in_proj_weight tensor(True)
period_predictor.transformer.decoder.layers.3.multihead_attn.in_proj_bias tensor(True)
period_predictor.transformer.decoder.layers.3.multihead_attn.out_proj.weight tensor(True)
period_predictor.transformer.decoder.layers.3.multihead_attn.out_proj.bias tensor(True)
period_predictor.transformer.decoder.layers.3.linear1.weight tensor(True)
period_predictor.transformer.decoder.layers.3.linear1.bias tensor(True)
period_predictor.transformer.decoder.layers.3.linear2.weight tensor(True)
period_predictor.transformer.decoder.layers.3.linear2.bias tensor(True)
period_predictor.transformer.decoder.layers.3.norm1.weight tensor(True)
period_predictor.transformer.decoder.layers.3.norm1.bias tensor(True)
period_predictor.transformer.decoder.layers.3.norm2.weight tensor(True)
period_predictor.transformer.decoder.layers.3.norm2.bias tensor(True)
period_predictor.transformer.decoder.layers.3.norm3.weight tensor(True)
period_predictor.transformer.decoder.layers.3.norm3.bias tensor(True)
period_predictor.transformer.decoder.layers.4.self_attn.in_proj_weight tensor(True)
period_predictor.transformer.decoder.layers.4.self_attn.in_proj_bias tensor(True)
period_predictor.transformer.decoder.layers.4.self_attn.out_proj.weight tensor(True)
period_predictor.transformer.decoder.layers.4.self_attn.out_proj.bias tensor(True)
period_predictor.transformer.decoder.layers.4.multihead_attn.in_proj_weight tensor(True)
period_predictor.transformer.decoder.layers.4.multihead_attn.in_proj_bias tensor(True)
period_predictor.transformer.decoder.layers.4.multihead_attn.out_proj.weight tensor(True)
period_predictor.transformer.decoder.layers.4.multihead_attn.out_proj.bias tensor(True)
period_predictor.transformer.decoder.layers.4.linear1.weight tensor(True)
period_predictor.transformer.decoder.layers.4.linear1.bias tensor(True)
period_predictor.transformer.decoder.layers.4.linear2.weight tensor(True)
period_predictor.transformer.decoder.layers.4.linear2.bias tensor(True)
period_predictor.transformer.decoder.layers.4.norm1.weight tensor(True)
period_predictor.transformer.decoder.layers.4.norm1.bias tensor(True)
period_predictor.transformer.decoder.layers.4.norm2.weight tensor(True)
period_predictor.transformer.decoder.layers.4.norm2.bias tensor(True)
period_predictor.transformer.decoder.layers.4.norm3.weight tensor(True)
period_predictor.transformer.decoder.layers.4.norm3.bias tensor(True)
period_predictor.transformer.decoder.layers.5.self_attn.in_proj_weight tensor(True)
period_predictor.transformer.decoder.layers.5.self_attn.in_proj_bias tensor(True)
period_predictor.transformer.decoder.layers.5.self_attn.out_proj.weight tensor(True)
period_predictor.transformer.decoder.layers.5.self_attn.out_proj.bias tensor(True)
period_predictor.transformer.decoder.layers.5.multihead_attn.in_proj_weight tensor(True)
period_predictor.transformer.decoder.layers.5.multihead_attn.in_proj_bias tensor(True)
period_predictor.transformer.decoder.layers.5.multihead_attn.out_proj.weight tensor(True)
period_predictor.transformer.decoder.layers.5.multihead_attn.out_proj.bias tensor(True)
period_predictor.transformer.decoder.layers.5.linear1.weight tensor(True)
period_predictor.transformer.decoder.layers.5.linear1.bias tensor(True)
period_predictor.transformer.decoder.layers.5.linear2.weight tensor(True)
period_predictor.transformer.decoder.layers.5.linear2.bias tensor(True)
period_predictor.transformer.decoder.layers.5.norm1.weight tensor(True)
period_predictor.transformer.decoder.layers.5.norm1.bias tensor(True)
period_predictor.transformer.decoder.layers.5.norm2.weight tensor(True)
period_predictor.transformer.decoder.layers.5.norm2.bias tensor(True)
period_predictor.transformer.decoder.layers.5.norm3.weight tensor(True)
period_predictor.transformer.decoder.layers.5.norm3.bias tensor(True)
period_predictor.transformer.decoder.norm.weight tensor(True)
period_predictor.transformer.decoder.norm.bias tensor(True)
classifier.l_network.0.weight tensor(True)
classifier.l_network.0.bias tensor(True)
classifier.l_network.2.weight tensor(True)
classifier.l_network.2.bias tensor(True)
classifier.p_network.0.weight tensor(True)
classifier.p_network.0.bias tensor(True)
classifier.p_network.2.weight tensor(True)
classifier.p_network.2.bias tensor(True)


I’m wondering if the bottleneck I calculate in here:
        sim_mat = -torch.matrix_power(torch.cdist(features, features), 2).unsqueeze(1)


may cause this.