Surprisingly the mask is slower!
mask = to.zeros((N, T,), dtype=to.bool, device=x.device)
idx = 0
for dur, count in zip(durs, counts):
    mask[idx: idx + count, :dur] = True
    idx += count
joined = (x * mask.unsqueeze(2).expand(-1, -1, x.size(2))).sum(dim=1)

is slower than
joined = to.zeros((N, self.repr_dim), dtype=to.float32, device=x.device)
idx = 0
for dur, count in zip(durs, counts):
    x_part = x[idx: idx + count, :dur]
    joined[idx: idx + count] = x_part.sum(dim=1)
    idx += count