I adapted yours to reproduce the result. The issue with the above code is that since world_size=1 it doesn’t trigger the proper code in nn.SyncBatchNorm (lines 528 to 548 of torch/nn/modules/batchnorm.py). This code triggers the error.
import torch
import torch.nn as nn
from torch.nn.parallel import DistributedDataParallel
import torch.distributed as dist
import argparse
from torch.cuda.amp import autocast, GradScaler
import os
import torch.multiprocessing as mp


class MyModel(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv = nn.Conv2d(3, 16, 3, 1, 1)
        self.bn = nn.BatchNorm2d(16, track_running_stats=False)

    def forward(self, x):
        with autocast():
            x = self.conv(x)
            x = self.bn(x)
            return x


def main(gpu):
    parser = argparse.ArgumentParser(description='fdsa')
    parser.add_argument("--local_rank", default=0, type=int)

    args = parser.parse_args()
    args.gpu = args.local_rank
    torch.cuda.set_device(args.gpu)

    rank = gpu
    device = f'cuda:{gpu}'

    dist.init_process_group(
        backend='nccl',
        init_method='env://',
        world_size=2,
        rank=rank)

    args.world_size = torch.distributed.get_world_size()

    model = MyModel()
    model = model.to(device)
    model = nn.SyncBatchNorm.convert_sync_batchnorm(model)

    model = DistributedDataParallel(
        model,
        device_ids=[rank],
        output_device=rank,
        broadcast_buffers=False,
        find_unused_parameters=False
    )

    scaler = GradScaler()
    criterion = nn.MSELoss()

    for i in range(2):
        model.zero_grad()
        x = torch.randn(2, 3, 16, 16).to(device)
        target = torch.randn(2, 16, 16, 16).to(device)
        with autocast():
            out = model(x)
            loss = criterion(out, target)

        scaler.scale(loss).backward()

    print('completed')

if __name__ == "__main__":
    os.environ["MASTER_ADDR"] = "localhost"
    os.environ["MASTER_PORT"] = "29500"
    mp.spawn(main, nprocs=2)