If I’m not mistaken, this code 1 will be used, which will launch a lot of GPU kernels (nsys shows 160045 cudaLaunchKernel calls) which should create a lot of CPU overhead.
To reduce the overhead caused by the kernel launches, you could try to use CUDA graphs as described here 3 and see if it would match your use case.
Example:
import torch
import torchaudio
import time

device = 'cuda'
nb_iters = 100

# Placeholder input used for capture
waveform = torch.rand([16000]).to(device)
a = torch.tensor([1.0, 0.0]).to(device)
b = torch.tensor([1.0, -0.97]).to(device)

# warmup
for _ in range(10):
    waveform = torchaudio.functional.lfilter(waveform, a, b)

# profile
torch.cuda.synchronize()
t0 = time.perf_counter()
for _ in range(nb_iters):
    waveform = torchaudio.functional.lfilter(waveform, a, b)
torch.cuda.synchronize()
t1 = time.perf_counter()

print('Eager, {}s/iter'.format((t1 - t0)/nb_iters))

# CUDA graphs

g = torch.cuda.CUDAGraph()

# Warmup before capture
s = torch.cuda.Stream()
s.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(s):
    for _ in range(3):
        waveform = torchaudio.functional.lfilter(waveform, a, b)
torch.cuda.current_stream().wait_stream(s)

# Captures the graph
# To allow capture, automatically sets a side stream as the current stream in the context
with torch.cuda.graph(g):
    waveform = torchaudio.functional.lfilter(waveform, a, b)

torch.cuda.synchronize()
t0 = time.perf_counter()
for _ in range(nb_iters):
    g.replay()
torch.cuda.synchronize()
t1 = time.perf_counter()

print('CUDA graphs, {}s/iter'.format((t1 - t0)/nb_iters))

Output:
Eager, 0.6360926309000934s/iter
CUDA graphs, 0.07975950390042272s/iter