non, did not change anything
Maybe it is because nvidia apex
model = load_model(hparams)

    optimizer = Ranger(model.parameters(), lr=hparams.learning_rate)

    criterion = Tacotron2Loss()

    logger = prepare_directories_and_logger(
        output_directory, log_directory, rank)

    train_loader, valset, collate_fn = prepare_dataloaders(hparams)

    iteration = 0
    epoch_offset = 0

    if hparams.fp16_run:
        from apex import amp
        model, optimizer = amp.initialize(model, optimizer, opt_level='O2')
    # Load checkpoint if one exists
    if os.path.isfile(checkpoint_path):
        model, optimizer, iteration = load_checkpoint(
            checkpoint_path, model, optimizer)
        iteration += 1  # next iteration is iteration + 1
        epoch_offset = max(0, int(iteration / len(train_loader)))
        if hparams.fp16_run:
            amp.load_state_dict(torch.load(
                checkpoint_path)['amp'])
    elif os.path.isfile(checkpoint_path_vanilla):
        model = warm_start_model(
            checkpoint_path_vanilla, model, hparams.ignore_layers)

    scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
        optimizer, hparams.epochs - hparams.epochs * hparams.epochs_annealing, eta_min=1e-6)

    if hparams.distributed_run:
        model = apply_gradient_allreduce(model)

    model.train()
    is_overflow = False
    # ================ MAIN TRAINNIG LOOP! ===================
    for epoch in range(epoch_offset, hparams.epochs):
        def set_dropout(model, drop_rate=0.1):
            for name, child in model.named_children():
                if isinstance(child, torch.nn.Dropout):
                    child.p = drop_rate
                set_dropout(child, drop_rate=drop_rate)
        if epoch <= 50:
            set_dropout(model, drop_rate=epoch / 100)
        print("Epoch: {}".format(epoch))
        start_epoch = time.perf_counter()
        for i, batch in enumerate(train_loader):
            start = time.perf_counter()
            model.zero_grad()
            x, y = model.parse_batch(batch)
            #y_pred = model(x)

            loss = criterion(model(x), y, x[-1])

            if hparams.distributed_run:
                reduced_loss = reduce_tensor(loss.data, n_gpus).item()
            else:
                reduced_loss = loss.item()
            if hparams.fp16_run:
                with amp.scale_loss(loss, optimizer) as scaled_loss:
                    scaled_loss.backward()
            else:
                loss.backward()

            if hparams.fp16_run:
                grad_norm = torch.nn.utils.clip_grad_norm_(
                    amp.master_params(optimizer), hparams.grad_clip_thresh)
                is_overflow = math.isnan(grad_norm)
            else:
                grad_norm = torch.nn.utils.clip_grad_norm_(
                    model.parameters(), hparams.grad_clip_thresh)

            optimizer.step()