KFrank:

torch.diag (float ('inf') * torch.ones (3))


Thank you for your timely help. Basically, I want to penalize the most similar pairs of each class. That’s why I want to get a similar loss which serves as a regularizing term. My total loss is like L_total = L_CE + L_sim. My questions as follows:

I want to know if this term can help to update the parameters of my network in the training process cause I did an assignment for model weights here weights = self.model.module.model.fc[-1].weight.
If I want to get a weighed similar loss, is the below code based on yours correct?

import torch
torch.__version__

torch.random.manual_seed (2020)
def cc_sim (weights, w_s):
    '''
    weights: weights from NN model
    w_s: weights for every entry in similarity matrix, w_s.size = (num_classes, num_classes)
    '''
    cos = torch.nn.CosineSimilarity(dim=1, eps=1e-6)
    num_iter = num_classes = weights.size(0)
    # similarity matrix
    sim_mat = torch.empty(0, num_iter)
    for j in range(num_iter):
        weights_i = weights[j, :].expand_as(weights)
        sim = cos(weights, weights_i)
        sim = torch.unsqueeze(sim, 0)
        sim_mat = torch.cat((sim_mat, sim), 0)
    sim_mat = sim_mat * w_s
    sim_mat = sim_mat - torch.diag(torch.diag(sim_mat))
    max_val = torch.max(sim_mat, dim=1).values
    loss_sim = (torch.sum(max_val) / num_classes**2)
    return loss_sim

def cc_simA (weights, w_s):
    '''
    weights: weights from NN model
    w_s: weights for every entry in similarity matrix, w_s.size = (num_classes, num_classes)
    '''
    cos = torch.nn.CosineSimilarity(dim=1, eps=1e-6)
    num_iter = num_classes = weights.size(0)
    # similarity matrix
    sim_mat = torch.empty(0, num_iter)
    for j in range(num_iter):
        weights_i = weights[j, :].expand_as(weights)
        sim = cos(weights, weights_i)
        sim = torch.unsqueeze(sim, 0)
        sim_mat = torch.cat((sim_mat, sim), 0)
    # element-wise multiply 
    sim_mat = sim_mat * w_s

    sim_mat = sim_mat - torch.diag(torch.diag(sim_mat))
    loss_sim = torch.mean (sim_mat)
    return loss_sim

def cc_simB (weights, w_s):
    '''
    weights: weights from NN model
    w_s: weights for every entry in similarity matrix, w_s.size = (num_classes, num_classes)
    '''
    cos = torch.nn.CosineSimilarity(dim=1, eps=1e-6)
    num_iter = num_classes = weights.size(0)
    # similarity matrix
    sim_mat = torch.empty(0, num_iter)
    for j in range(num_iter):
        weights_i = weights[j, :].expand_as(weights)
        sim = cos(weights, weights_i)
        sim = torch.unsqueeze(sim, 0)
        sim_mat = torch.cat((sim_mat, sim), 0)
    # element-wise multiply 
    sim_mat = sim_mat * w_s
    sim_mat = sim_mat - torch.diag (float ('inf') * torch.ones (num_classes))
    max_val = torch.max(sim_mat, dim=1).values
    loss_sim = (torch.sum(max_val) / num_classes**2)
    return loss_sim

if __name__ == '__main__':
    num_classes = nDim = 10
    weights = torch.randn((num_classes, nDim), requires_grad=True)
    w_s = torch.FloatTensor(num_classes, num_classes).uniform_(0, 1) 
    loss = cc_simB(weights, w_s)
    loss.backward()
    print('loss item:', loss)
    print('grad of weights:\n')
    print(weights.grad)


Thanks again!