@holiv you can pre-allocate a larger Tensor, and then just copy into a slice.
max_output_size = 10
output_cat = None

start=time.time()
for i in range(2):
        hx= rnn(input[:,i,:], (hx_0))
        output.append(hx)

        if output_cat is None:
            output_cat_size = list(hx.size())
            output_cat_size.insert(1, max_output_size)
            output_cat = torch.empty(*output_cat_size, dtype=hx.dtype, device=hx.device)

        output_cat[:, i] = hx
        hx_0 = operation(output_cat[:, 0:i])

print(output_cat)