I cannot reproduce the 30ms using the PyTorch 1.8.0 binaries:
import torch
import torch.nn.functional as F
import time

y = torch.randint(0, 10, (128,), device='cuda')

# warmup
for _ in range(10):
    y_one_hot = F.one_hot(y, num_classes=10)

nb_iters = 1000
torch.cuda.synchronize()
t0 = time.time()
for _ in range(nb_iters):
    y_one_hot = F.one_hot(y, num_classes=10)
torch.cuda.synchronize()
t1 = time.time()
print('{:.3f}us/iter'.format(((t1 - t0)/nb_iters)*1e6))

Output
83.642us/iter