Update: The SHA-RNN codebase, which claimed to be very memory and computationally efficient, just uses torch.cat. I have been decieved.

  
      github.com
  
  
    Smerity/sha-rnn/blob/master/model.py#L209 2

    #x = x + z.sum(dim=-2)

    h = h + x if self.residual else x.float()

focus, new_mem = None, []
if self.attn is not None:
    mh = self.lnmem(h)
    h = self.lnmid(h)

    if mem is not None:
        bigh = torch.cat([mem, mh], dim=0)
    else:
        bigh = mh
    new_mem = bigh[-len(pe):]

    q, k = h, bigh

    x, focus = checkpoint(self.attn, q, k, bigh, attn_mask)
    #x, focus = tcheckpoint(self.attn, q, k, bigh, attn_mask)
    x = self.drop(x)
    h = x + h