For what it’s worth - I reproed this on Tesla M60 GPU and saw the same behavior as OP - that with autocast enabled forward pass is marginally slower than without.
# fp16=True
  92.23 ms
  1 measurement, 100 runs , 1 thread
# fp16=False
  79.00 ms
  1 measurement, 100 runs , 1 thread

I also tried including backward pass, and observed the times were very similar:
model = BertForSequenceClassification.from_pretrained("bert-base-uncased").to(0)

tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
inputs = tokenizer("Hello, my dog is cute", return_tensors="pt").to(0)
labels = torch.tensor([1]).unsqueeze(0).to(0)

def autocast_backward(model, inputs, labels, fp16=False):
    with autocast(enabled=fp16):
        outputs = model(**inputs, labels=labels)
        loss = outputs.loss
        loss.backward()

t0 = benchmark.Timer(
    stmt="autocast_backward(model, x, False)",
    setup="from __main__ import autocast_backward",
    globals={"inputs": inputs, "model": model, "labels": labels,},
)

t1 = benchmark.Timer(
    stmt="autocast_backward(model, x, True)",
    setup="from __main__ import autocast_backward",
    globals={"inputs": inputs, "model": model, "labels": labels,},
)

print(t1.timeit(100))
print(t0.timeit(100))

gave
# fp16=True
  304.61 ms
  1 measurement, 100 runs , 1 thread
# fp16=False
  304.99 ms
  1 measurement, 100 runs , 1 thread