–  EDIT: some final answers 2. in short, cuda streams are very limited.
hi,
pytorch doc defines cuda streams as A CUDA stream is a linear sequence of execution that belongs to a specific device.

i assume by linear sequence, they meant a set of cuda instructions without control statements. (?)
what would happen if the sequence is not linear?
what would happen if the instructions contain mixed instructions: once for cuda, other for cpu such as print(!!!) or more cpu stuff. so, if your set of instructions are mixed and not linear, cuda streams dont seem to be a solution to speedup.
so fare, i am unable to see any speedup using streams over linear cuda sequence. any explanations? this 3 suggests that when the cuda instructions are run in a time shorter that time required for cpu to start the next stream, you wont see any speedup.  i modified the code in order to slow down a stream in order to give enough time to the cpu to start the other one, but i still dont see any speedup.

thanks
this is the run time of the below code, and it uses only 3gb/16gb of gpu memory:
$ CUDA_LAUNCH_BLOCKING=1 python streamer.py 
time linear:          276498.625ms
time concurrent:  277744.0625ms


code streamer.py:
import time
import torch
import torch.nn as nn

def run(iters=10, streams=False):
    device = torch.device(1)

    s1 = torch.cuda.Stream(device=device)
    s2 = torch.cuda.Stream(device=device)
    x = torch.rand(size=(1024 * 10, 1024 * 10)).to(device)
    w1 = torch.rand(size=(1024 * 10, 1024 * 10)).to(device)
    w2 = torch.rand(size=(1024 * 10, 1024 * 10)).to(device)

    def op():
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)
        x.matmul(w1)

        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)
        x.matmul(w2)

    torch.cuda.synchronize()

    for i in range(iters):
        torch.cuda.nvtx.range_push('iter{}'.format(i))

        if streams:

            with torch.cuda.stream(s1):
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)
                x.matmul(w1)

            with torch.cuda.stream(s2):
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)
                x.matmul(w2)

        else:
            op()

        torch.cuda.nvtx.range_pop()

    torch.cuda.synchronize()


if __name__ == '__main__':
    # warmup
    run()

    torch.cuda.cudart().cudaProfilerStart()
    start_event = torch.cuda.Event(enable_timing=True)
    end_event = torch.cuda.Event(enable_timing=True)
    start_event.record()
    run(streams=False)
    end_event.record()
    elapsed_time_ms = start_event.elapsed_time(end_event)
    print('time linear: {}ms'.format(elapsed_time_ms))
    torch.cuda.cudart().cudaProfilerStop()

    start_event.record()
    run(streams=True)
    end_event.record()
    elapsed_time_ms = start_event.elapsed_time(end_event)
    print('time concurrent: {}ms'.format(elapsed_time_ms))