I added torch.cuda.synchronize() as you said (that could have been the problem some of the time) but the assertion still fails on some random batches.  I suspect that idx is getting mixed up between the for loop and the with code block. And that will make my concat happen in the wrong order (since I’m using idx to sort the intermediate results)
def forward(self, emb):
  #old way   
  tmp = [cnn(emb).squeeze() for cnn in self.cnns]
  seq_representation = torch.cat(tmp, dim=1)

  #new way
  stream_tmp = []
  streams = [(idx, torch.cuda.Stream()) for idx, cnn in enumerate(self.cnns)]
  for idx, s in streams:
    with torch.cuda.stream(s):
      cnn = self.cnns[idx]     #<--- how to ensure idx is in sync with the idx in for loop?
      stream_tmp.append((idx, cnn(emb).squeeze()))
  torch.cuda.synchronize() # added synchronize
  stream_tmp = [t for idx, t in sorted(stream_tmp)]
  seq_representation_stream = torch.cat(stream_tmp, dim=1)  
  
  #comparing the two
  diff = abs(seq_representation_stream - seq_representation).sum().data[0])
  print(diff)
  assert diff == 0.0
  return seq_representation