I do not think is related with no_grad(). But every time I add in my code the lines between {{}} I get nan in my losses after some epochs. Sorry for the long code. Training loop is the same as testing loop in validation, the only extras are the backpropagation procedure.
                {{
                with torch.no_grad():                        
        
                        recommender_encoder.eval()
                        recommender_decoder.eval()
        
                        role='Testing'
                        rouge_score_test,rating_loss_test = validation(recommender_encoder,recommender_decoder,epoch,role)
                        
                        role='Training'
                        rouge_score_train,rating_loss_train = validation(recommender_encoder,recommender_decoder,epoch,role)
                        }}

#test and make the sentences
def validation(recommender_encoder,recommender_decoder,epoch,role):

        #print("\n\n--------------------------- Loaded model from disk ---------------------------\n\n")
        
        ''' define the decay function in order to control the trade off between feeding the model with it's output or the real output | #initialize the decay '''
        teacher_forcing_ratio_test = 1 #because we want the decoder to feed itself only with it's own outputs       
        
        if (role=='Testing'):
                #define the datalodaer
                dataloader_test = dataset_loader.Dataset_yelp('data/REAL_minibatch_testing.pkl', transform=None) #output_space = 8171
        elif (role=='Training'):
                #define the datalodaer
                dataloader_test = dataset_loader.Dataset_yelp('data/REAL_minibatch_training.pkl', transform=None) #output_space = 8171
        elif (role=='Development'):      
                #define the datalodaer
                dataloader_test = dataset_loader.Dataset_yelp('data/REAL_minibatch_development.pkl', transform=None) #output_space = 8171       
        
        
        batch_size_test = 32
        train_sampler = ShuffleBatchSampler(SortedSampler(dataloader_test,sort_key=lambda i: len(i[4])),batch_size=batch_size_test,drop_last=False,shuffle=False)
        
        
        #define some proparties for the dataloader
        loader_params_test = {'num_workers':4, 'pin_memory':True, 'collate_fn':custom_collate, 'batch_sampler':train_sampler}
        dataloader_test = torch.utils.data.DataLoader(dataloader_test, **loader_params_test)
        
        
        #define the loss function for the review prediction
        weights_test = initialize_loss_weights()
        class_weights_test = torch.FloatTensor(weights_test).cuda()
        criterion_review_test = torch.nn.CrossEntropyLoss(class_weights_test).cuda() #to put the calcualtion into GPU
        
        
        #for the review maybe we use Negative Log likelihood error
        #criterion_review = torch.nn.NLLLoss().cuda()        
        #define the loss function for the rating prediction, we want to use RMSE so we have to use torch.sqrt(criterion(predicted, gound_truth))
        criterion_rating_test = torch.nn.MSELoss().cuda()

        overall_review_loss_test = []
        overall_rating_loss_test = []
        
        overall_rouge_score_test = []

        
        #iterate all the batches
        for index_test, (user_reviews_test, product_reviews_test, neighbourhood_reviews_test, product_ratings_test, text_reviews_test, rating_reviews_test, decoder_inputs_test) in enumerate(dataloader_test):
        #for index, (user_reviews, product_reviews, neighbourhood_reviews, product_ratings, text_reviews, rating_reviews, decoder_inputs) in enumerate([first]):
                
                #change some properties of the input. Move it to GPU and make it float tensor
                user_reviews_test = user_reviews_test.type(torch.FloatTensor)
                product_reviews_test = product_reviews_test.type(torch.FloatTensor)
                #format the ratings
                product_ratings_test = product_ratings_test.type(torch.FloatTensor)
                #take the output size, this is different for every batch
                output_length_test =  text_reviews_test.shape[1]
                
                #print('User Input reviews',user_reviews.shape)
                #print('Product input reviews',product_reviews.shape)
                #print('Product reviews ratings',product_ratings.shape)
                #print('Target review',text_reviews.shape)
                #print('Target rating',rating_reviews.shape)
                #print('Batch:',index)
                #print('---------- new ----------')
                
                #initialize mask to use it in the prediction of rating
                text_reviews_mask_test = text_reviews_test.view(-1).type(torch.LongTensor).to(device1)
                mask_test = initialize_mask(text_reviews_mask_test)
                
                user_reviews_test = user_reviews_test.to(device1)
                product_reviews_test = product_reviews_test.to(device1)
                product_ratings_test = product_ratings_test.to(device1)
                
                #run the encoder
                overall_context_vector_encoder_test, overall_attention_weights_test, decoder_hidden_test, user_h_test, product_h_test, user_lstm_output_test, product_lstm_output_test = recommender_encoder(user_reviews_test, product_reviews_test)
                
                
                #activations_loss, activations, rating_prediction, repetitiveness_loss_back = encoder_decoder_parallel(user_reviews, product_reviews, output_length, product_ratings, decoder_inputs, mask, teacher_forcing_ratio)
                
                #overall_context_vector_encoder = overall_context_vector_encoder.to(self.device1)
                #decoder_hidden = (decoder_hidden[0].to(self.device1),decoder_hidden[1].to(self.device1))
                #overall_attention_weights = overall_attention_weights.to(self.device1)
                decoder_inputs_test = decoder_inputs_test.to(device1)
                
                #run the decoder
                activations_loss_test, rating_prediction_test, repetitiveness_loss_back_test, target_length_test, decoder_h_c_list_test = recommender_decoder(overall_context_vector_encoder_test, decoder_hidden_test, decoder_inputs_test, output_length_test, overall_attention_weights_test, product_ratings_test, mask_test, teacher_forcing_ratio_test)
                

                '''
                #----------------------------------------------------------------------------------------> TEXT LOSS
                #replace Nan values with 0s
                #activations_loss[activations_loss==float('nan')] = 0
                activations_loss = activations_loss.view(-1, output_space)
                
                print(activations_loss,text_reviews)
                
                review_loss = criterion_review(activations_loss, text_reviews)
                overall_review_loss.append(review_loss.item())
                '''
                #----------------------------------------------------------------------------------------> RATING LOSS
                #move target to GPU, and change it's type
                rating_reviews_test = rating_reviews_test.to(device1)
                rating_prediction_test = rating_prediction_test.view(-1)
                rating_prediction_test = rating_prediction_test.type(torch.DoubleTensor).to(device1)
                #keep record of the rating prediction loss
                rating_loss_test = criterion_rating_test(rating_reviews_test, rating_prediction_test).type(torch.FloatTensor).to(device1)
                overall_rating_loss_test.append(rating_loss_test.item())
                
                #format the ratings
                #rating_prediction = rating_prediction.squeeze(1).squeeze(1)
                
                #iterate every sample in the batch
                for sample_test in range(activations_loss_test.shape[0]):
                        predicted_sentence_test = []
                        ground_truth_test = []
                        target_reviews_length_test = activations_loss_test[sample_test].shape[0]
                        for word_test in range(activations_loss_test[sample_test].shape[0]):  
                                #find the position of the max element, which represents the word ID
                                predicted_index_test = torch.argmax(activations_loss_test[sample_test][word_test]).item()
                                ground_truth_test.append(text_reviews_test[sample_test][word_test].item())
                                predicted_sentence_test.append(predicted_index_test)
                                
                                
                        ground_truth_rating_test = rating_reviews_test[sample_test].item()
                        rating_prediction_item_test = rating_prediction_test[sample_test].item()
                        count_predictions_test=1
                        
                        
                        #for every predicted sentence
                        print_sentence_test = False
                        max_review_test = 500
                        rouge_score_test = evaluation.make_sentence_one_by_one(predicted_sentence_test, ground_truth_test, target_reviews_length_test, tokenizer, role, count_predictions_test, rating_prediction_item_test, ground_truth_rating_test,print_sentence_test,max_review_test)
                        overall_rouge_score.append(rouge_score)

                return sum(overall_rouge_score_test)/len(overall_rouge_score_test), sum(overall_rating_loss_test)/len(overall_rating_loss_test)