I confirm that I can reproduce this locally.
Hey @osalpekar, do you know if we miss anything here? Would I be correct if I assume the following code is expected to abort the op in this case?

  
      github.com
  
  
    pytorch/pytorch/blob/ecb88c5d11895a68e5f20917d27a0debbc0f0697/torch/lib/c10d/ProcessGroupNCCL.cpp#L301-L335 5

if (blockingWait_) {
  // Use the passed in timeout if provided, otherwise use the default
  // opTimeout for each WorkNCCL object.
  std::chrono::milliseconds workTimeout =
      timeout == kNoTimeout ? opTimeout_ : timeout;
  // Wait for the operation to complete.
  while (!isCompleted()) {
    auto currentTimepoint = std::chrono::steady_clock::now();
    if (std::chrono::duration_cast<std::chrono::milliseconds>(
            currentTimepoint - workStartTime_) > workTimeout) {
      // When operation times out due to some errors that are not
      // detected by nccl communicators, ncclCommWatchdog can not check this
      // time out error and thus can not abort ncclComms accordingly.
      // So explicitly abort ncclComms here before throwing this timed out
      // exception to users, after this, ncclCommWatchdog can detect nccl
      // communicators are aborted and clean up devNCCLCommMap_ accordingly.
      // if throwing timed out excepiton without aborting nccl communicators
      // here, it was observed that CUDA GPU will have 100% utilization and
      // can not run new events successfully.
      for (const auto& ncclComm : ncclComms_) {


  This file has been truncated. show original