As batch_spk_centroid is local variable to the function calc_cosine_sim_div, I think maintaining the computational graph for the batch_spk_centroid is not required (please correct me if I am wrong.) So I have added with torch.no_grad() before the for loop begins. Below code is working perfectly.
def calc_cosine_sim_div(self, batch_embedding, batch_label):
        batch_spk_centroid = torch.mean(batch_embedding, dim = 1)
        cos_sim_matrix = []
        with torch.no_grad():
            for index1, spk_embedding in enumerate(batch_embedding):
                for index2, utt_embedding in enumerate(spk_embedding):
                    batch_spk_centroid[index1] = ((batch_spk_centroid[index1].clone() * self.hp.train.num_utt_per_spk ) - batch_embedding[index1][index2] ) / (self.hp.train.num_utt_per_spk - 1)
                    cos_sim_matrix.append(self.cos_sim(utt_embedding.repeat(batch_embedding.shape[0],1), batch_spk_centroid))
                    batch_spk_centroid[index1] = ((batch_spk_centroid[index1].clone() * (self.hp.train.num_utt_per_spk-1) ) + batch_embedding[index1][index2] ) / self.hp.train.num_utt_per_spk
        cos_sim_matrix = torch.stack(cos_sim_matrix).reshape(batch_embedding.shape[0],batch_embedding.shape[1],batch_embedding.shape[0])
        return cos_sim_matrix