The timing sounds weird.
Are you sure you are seeing this slowdown from hooks?
I would assume to see some overhead, but yours seems to be really large.
I’m seeing a ~4% slowdown using this dummy code:
model = models.resnet50()
model.cuda()

x = torch.randn(32, 3, 224, 224).cuda()

# warmup
for _ in range(50):
    out = model(x)

nb_iters = 100    
torch.cuda.synchronize()
t0 = time.time()

for _ in range(nb_iters):
    out = model(x)
    
torch.cuda.synchronize()
t1 = time.time()

print('{}s/iter'.format((t1 - t0)/nb_iters))


# Use hooks
def my_hook(m, input, output):
    output = output * 100
    return output
    
for child in model.children():
    child.register_forward_hook(my_hook)


# warmup
for _ in range(50):
    out = model(x)

nb_iters = 100    
torch.cuda.synchronize()
t0 = time.time()

for _ in range(nb_iters):
    out = model(x)
    
torch.cuda.synchronize()
t1 = time.time()

print('{}s/iter'.format((t1 - t0)/nb_iters))