github.com
  
  
    huggingface/transformers/blob/master/src/transformers/trainer.py#L789 76

    # last step in epoch but step is always smaller than gradient_accumulation_steps
    steps_in_epoch <= self.args.gradient_accumulation_steps
    and (step + 1) == steps_in_epoch
):
    if self.args.fp16 and _use_native_amp:
        self.scaler.unscale_(self.optimizer)
        torch.nn.utils.clip_grad_norm_(model.parameters(), self.args.max_grad_norm)
    elif self.args.fp16 and _use_apex:
        torch.nn.utils.clip_grad_norm_(amp.master_params(self.optimizer), self.args.max_grad_norm)
    else:
        torch.nn.utils.clip_grad_norm_(model.parameters(), self.args.max_grad_norm)

    if is_torch_tpu_available():
        xm.optimizer_step(self.optimizer)
    elif self.args.fp16 and _use_native_amp:
        self.scaler.step(self.optimizer)
        self.scaler.update()
    else:
        self.optimizer.step()

    self.lr_scheduler.step()



  
  
    
    
  
  


I know that gradient clipping is useful for preventing exploding gradients, is this is reason why it is there by default? Or does this improve overall model training quality?
Why is norm clipping used instead of the alternatives?