I confirm that I can reproduce the same behavior 3 CUDA process works but 4 does not work, with 15 CPU processes.

Could you please guide me towards why this is happening and how it can be resolved?

The difference between 1.9 and 1.7 is that we introduced RPC CUDA RMDA in v1.9. Direct Device-to-Device Communication with TensorPipe CUDA RPC — PyTorch Tutorials 1.9.0+cu102 documentation 3
So the first thing I tried is setting os.environ["CUDA_VISIBLE_DEVICES"] = "" in cpu_process_initialization. But I hit the following error:
RuntimeError: In create at tensorpipe/common/cuda_lib.h:117 "lib.init(0)(100) CUDA_ERROR_NO_DEVICE (no CUDA-capable device is detected)"

@lcw any opinion on whether we should let TensorPipe or TensorPipe_Agent to detect the CUDA device runtime availability? (Created an issue to track: init_rpc fails after setting CUDA_VISIBLE_DEVICES env var to "" · Issue #60578 · pytorch/pytorch · GitHub 2)
Then I played with the number of processes, it looks like it crashes as long as CUDA process + CPU process > 18. (i.e., 8 CUDA process + 10 CPU process also works). I don’t recall there is any hard limit on the number of processes in RPC. @lcw Does TensorPipe has restrictions on that?
Below is the repro I used locally.
import os
import time
import torch
#torch.multiprocessing.set_sharing_strategy('file_system')
import torch.multiprocessing as mp
import torch.distributed.rpc as rpc

no_of_saver_processes = 10
#world_size = torch.cuda.device_count()
world_size = 8

def cpu_process_initialization(rank):
    os.environ['MASTER_ADDR'] = 'localhost'
    os.environ['MASTER_PORT'] = '9867'
    os.environ["CUDA_VISIBLE_DEVICES"] = f"{rank % torch.cuda.device_count()}"
    rpc.init_rpc(f"{rank}",
                 rank=rank,
                 world_size = world_size + no_of_saver_processes,
                 backend=rpc.BackendType.TENSORPIPE,
                 rpc_backend_options=rpc.TensorPipeRpcBackendOptions(rpc_timeout=6000,
                                                                     init_method='env://')
                 )
    print(f"Started CPU process {rank}")
    print(f"Process {rank}: avaialable device {torch.cuda.current_device()}")
    # Do something rather than sleeping example disk or cpu bound operations
    time.sleep(30)
    rpc.shutdown()
    return

def cuda_process_initialization(rank):
    #torch.cuda.set_device(rank)
    os.environ["CUDA_VISIBLE_DEVICES"] = f"{rank}"
    os.environ['MASTER_ADDR'] = 'localhost'
    os.environ['MASTER_PORT'] = '9867'
    rpc.init_rpc(f"{rank}",
                 rank=rank,
                 world_size=world_size + no_of_saver_processes,
                 backend=rpc.BackendType.TENSORPIPE,
                 rpc_backend_options=rpc.TensorPipeRpcBackendOptions(#num_send_recv_threads=args.world_size*3,
                                                                     rpc_timeout=6000,
                                                                     init_method='env://')
                 )
    print(f"Started CUDA process on gpu {rank}")
    # Do some cuda operations
    print(f"Process {rank}: avaialable device {torch.cuda.current_device()}")
    time.sleep(30)
    rpc.shutdown()
    return

if __name__ == "__main__":
    mp.set_start_method('spawn', force=True)
    trainer_processes = mp.spawn(cuda_process_initialization,
                                 nprocs=world_size,
                                 join=False)
    cpu_processes = []
    for rank in range(world_size,world_size+no_of_saver_processes):
        p = mp.Process(target=cpu_process_initialization,
                       args=(rank,))
        p.start()
        cpu_processes.append(p)
    for p in cpu_processes: p.join()
    trainer_processes.join()