You could still scale the gradients while accumulating them inside the loop. Something like the second approach here could work:
model = models.resnet18()
criterion = nn.CrossEntropyLoss()

x = torch.randn(100, 3, 224, 224)
y = torch.randint(0, 1000, (100,))

# 1
loss = 0.
for x_, y_ in zip(x, y):
    x_ = x_.unsqueeze(0)
    y_ = y_.unsqueeze(0)
    output = model(x_)
    batch_loss = criterion(output, y_)
    loss += batch_loss
             
loss = loss/len(x)
params = [p for p in model.parameters() if p.requires_grad]
grad_ref = list(torch.autograd.grad(loss, params))


# 2
grad = None
params = [p for p in model.parameters() if p.requires_grad]
for x_, y_ in zip(x, y):
    x_ = x_.unsqueeze(0)
    y_ = y_.unsqueeze(0)
    output = model(x_)
    batch_loss = criterion(output, y_)
    if grad is None:
        grad = [g/len(x) for g in list(torch.autograd.grad(batch_loss, params))]
    else:
        tmp = list(torch.autograd.grad(batch_loss, params))
        grad = [g + t/len(x) for g, t in zip(grad, tmp)]

for g, g_ref in zip(grad, grad_ref):
    print((g - g_ref).abs().max())

You could also scale the batch_loss by len(x) alternatively.