I’m not sure, how the FP32 code snippet is used, but note that you are not zeroing out the gradients.
However, even after adding optim.zero_grad() neither the FP32 nor the AMP model converges nicely:
model = nn.Sequential(nn.Conv2d(in_channels=3, out_channels=3, kernel_size=3, stride=1, padding=1),
                       nn.Conv2d(in_channels=3, out_channels=1, kernel_size=1, stride=1, padding=0))
model = model.cuda().train()

scaler = torch.cuda.amp.GradScaler()

optim = torch.optim.SGD(model.parameters(), lr=1e-5)
image = torch.randn(1, 3, 24, 24).cuda()
mask = torch.empty(1, 1, 24, 24).uniform_().cuda()
loss = nn.BCEWithLogitsLoss()

use_amp = False

for i in range(1000):
    optim.zero_grad()
    with torch.cuda.amp.autocast(enabled=use_amp):
        out = model(image)
        l = loss(out, mask)
    if use_amp:
        scaler.scale(l).backward()
        scaler.step(optim)
        scaler.update()
    else:
        l.backward()
        optim.step()
        
    print(l)