I would like to know if these lines of code arranged correctly:
lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, len(dataloader))               
for batch_i, (_, imgs, targets) in enumerate(dataloader):
    optimizer.zero_grad() 
    with torch.cuda.amp.autocast(enabled=use_amp):
        imgs = Variable(imgs.type(Tensor))
        targets = Variable(targets.type(Tensor), requires_grad=False)
        loss = model(imgs, targets)
    
    scaler.scale(loss).backward()

    # accumulate gradient for x batches before optimizing
    if ((batch_i + 1) % accumulated_batches == 0) or (batch_i == len(dataloader) - 1):
        # Unscales the gradients of optimizer's assigned params in-place
        scaler.unscale_(optimizer)

        # Since the gradients of optimizer's assigned params are now unscaled, clips as usual.
        # You may use the same value for max_norm here as you would without gradient scaling.
        torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=0.1)
        scaler.step(optimizer)
        lr_scheduler.step()
        scaler.update()
        optimizer.zero_grad() # set_to_none=True here can modestly improve performance

Thanks