With the hint about cudnn.benchmark, I have a minimal example that reproduces the behavior, if anyone is curious
import torch
import torch.nn as nn


def pprint_snapshot():
    s = torch.cuda.memory_snapshot()
    for seg in s:
        print("%7.2f | %7.2f MB - %s" % (
            seg["active_size"] / 1000000., seg["total_size"] / 1000000., seg["segment_type"]))
        for b in seg["blocks"]:
            print("    %7.2f MB - %s" % (b["size"] / 1000000., b["state"]))


torch.backends.cudnn.benchmark = True
conv = nn.Conv2d(128, 128, 3, padding=1).to("cuda:0")
input = torch.randn((2, 128, 90, 160), dtype=torch.float32).to("cuda:0")
pprint_snapshot()
output = conv(input)
print("output done")
pprint_snapshot()