Hey @Yanan
In this case, the tensor will be directly copied from device to device using cudaMemcpyAsync. The C++ implementation is linked below:

  
      github.com
  
  
    pytorch/pytorch/blob/89c0efb30b5bad49e717c1b1a797bac3c62e8b7e/aten/src/ATen/native/cuda/Copy.cu#L56-L67 12

if (memcpy_eligible) {
  void *dst = iter.data_ptr(0);
  void *src = iter.data_ptr(1);
  size_t size = numel * iter.element_size(0);
  if (src != dst || src_device != dst_device) {
    // Perform the copy
    AT_CUDA_CHECK(cudaMemcpyAsync(
        dst, src, size,
        cudaMemcpyDeviceToDevice,
        copy_stream));
  }
} else {