I used something similar to the following code -
import os
import multiprocessing as mp
import time
import torch

def launch_proc(gpu_id):
    os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu_id)
    import torch
    #Allocate memory on visible GPU
    tensor = torch.Tensor((10**8)*[gpu_id]).cuda()
    time.sleep(10) #wait for 10 seconds 

if __name__ == '__main__':
    num_gpus = torch.cuda.device_count()
    ctx = mp.get_context('spawn')
    with ctx.Pool(processes=num_gpus) as pool:
        pool.starmap(launch_proc, [(i,) for i in range(num_gpus)])


The main change was to use the spawn context and enclose the GPU launch processes within if __name__ == '__main__'.