ptrblck:

torch.utils.benchmark


Still not seeing speed up.
import torch
from torch.cuda.amp import autocast
print(torch.__version__)

!nvidia-smi

from transformers import AutoModel
model = AutoModel.from_pretrained('bert-base-uncased').to(1)
model.train()

x=torch.randint(0,1000, (8,128)).to(1)

def autocast_forward(model, x, fp16 = False):
    with autocast(enabled = fp16):
        return model.forward(x)



import torch.utils.benchmark as benchmark

t0 = benchmark.Timer(
    stmt='autocast_forward(model, x, False)',
    setup='from __main__ import autocast_forward',
    globals={'x': x, 'model':model})

t1 = benchmark.Timer(
    stmt='autocast_forward(model, x, True)',
    setup='from __main__ import autocast_forward',
    globals={'x': x, 'model':model})

print(t0.timeit(100))
print(t1.timeit(100))

<torch.utils.benchmark.utils.common.Measurement object at 0x7f9e30ccd520>
autocast_forward(model, x, False)
  19.95 ms
  1 measurement, 100 runs , 1 thread
<torch.utils.benchmark.utils.common.Measurement object at 0x7f9d9fa651c0>
autocast_forward(model, x, True)
  23.53 ms
  1 measurement, 100 runs , 1 thread