Given the implementation below, it does not seem ProcessGroupMPI uses any dedicated CUDA streams. So I would assume it’s fully delegated to MPI’s implementation?

  
      github.com
  
  
    pytorch/pytorch/blob/f314d9a0774062a20015ae522d33eadd45293328/torch/lib/c10d/ProcessGroupMPI.cpp#L791-L814 4

std::shared_ptr<ProcessGroup::Work> ProcessGroupMPI::send(
    std::vector<at::Tensor>& tensors,
    int dstRank,
    int tag) {
  checkSingleTensor(tensors);

  auto& tensor = tensors[0];
  MPI_Request request = MPI_REQUEST_NULL;

  {
    c10::DeviceGuard guard(tensor.device());
    std::unique_lock<std::mutex> globalLock(pgGlobalMutex_);
    MPI_CHECK(MPI_Isend(
        tensor.data_ptr(),
        tensor.numel(),
        mpiDatatype.at(tensor.scalar_type()),
        dstRank,
        tag,
        pgComm_,
        &request));


  This file has been truncated. show original