I tried to run my code with and without the CUDAStreamGuard commented out and the runtime is the same:
std::vector<at::cuda::CUDAStream> streams;
        streams.emplace_back(at::cuda::getStreamFromPool());
        streams.emplace_back(at::cuda::getStreamFromPool());
        streams.emplace_back(at::cuda::getStreamFromPool());

    measuretime();
    {
        torch::mm_out(X, A, B);
        at::cuda::CUDAStreamGuard g0(streams[0]);
        torch::mm_out(Y, A, C);
        at::cuda::CUDAStreamGuard g1(streams[1]);
        torch::mm_out(Z, A, D);
        at::cuda::CUDAStreamGuard g2(streams[2]);
        torch::mm_out(W, A, E);
    }
    measuretime();

What am I doing wrong