To avoid repetition you can use gpu index when saving files. Something like the following:
def save_tensors(gpu, total_gpus):
    torch.cuda.set_device(gpu)
    torch.distributed.init_process_group(backend = 'nccl', init_method='env://', world_size=total_gpus, rank=gpu)
    for i in range(100):
        tensor = torch.rand(2,3)
        torch.save(tensor, f'tensor_{i}_gpu_{gpu}.pt')
def main():
    gpu_count = torch.cuda.device_count()
    os.environ['MASTER_ADDR'] = '127.0.0.1'
    os.environ['MASTER_PORT'] = '1234'
    torch.multiprocessing.spawn(save_tensors, nprocs = gpu_count, args = (gpu_count, ))