That’s up to whether you have set the current device.
def worker(gpu, *args):
    ...
    torch.cuda.set_device(gpu) # <- This line makes the difference
    if gpu == 0:
        prob = torch.rand(1).cuda()
        print(gpu, prob)
    else:
        prob = torch.zeros(1).cuda()
        print(gpu, prob)

# output
0 tensor([0.6916], device='cuda:0')
1 tensor([0.], device='cuda:1') # <- on correct device