Well, I meant to show this property changing method works.
This is the complete solution I came up with for my main question:
import torch
import torch.distributed as dist
import torch.multiprocessing as mp
import torch.utils.data.distributed
from torch.utils.data import Dataset, DataLoader
import torchvision.transforms as T
import numpy as np
import argparse




class MyDataset(Dataset):
    def __init__(self):
        self.nb_samples = 300
        self.data = np.array([x + .2 for x in range(0, self.nb_samples)])
        self.label = np.array([x + .1 for x in range(0, self.nb_samples)])
        self.index = np.array([x  for x in range(0, self.nb_samples)])

    def __getitem__(self, index):
        if torch.is_tensor(index):
            index = index.tolist()
        return torch.tensor(self.data[index]), torch.tensor(self.label[index]), torch.tensor(self.index[index])

    def __len__(self):
        return self.nb_samples

    def get_sample(self, index):
        print(f'INDEXs: {index}')
        if torch.is_tensor(index):
            index = index.flatten().tolist()
        print(f'INDEX: {index}')
        return torch.tensor(self.data[index]), torch.tensor(self.label[index]), torch.tensor(self.index[index])


def main(device, args):
    ngpus_per_node = torch.cuda.device_count()
    args.world_size = torch.cuda.device_count() * args.world_size
    mp.spawn(main_worker, nprocs=ngpus_per_node, args=(ngpus_per_node, args))


def main_worker(gpu, ngpus_per_node, args):
    args.gpu = gpu
  
    args.rank = args.rank * ngpus_per_node + gpu
    dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url,
                            world_size=args.world_size, rank=args.rank)
    torch.cuda.set_device(args.gpu)
    args.batch_size = int(args.batch_size / ngpus_per_node)
    args.workers = int((args.workers + ngpus_per_node - 1) / ngpus_per_node)

    dataset = MyDataset()
    sampler = torch.utils.data.distributed.DistributedSampler(dataset)
    loader = DataLoader(dataset,
                        num_workers=args.workers,
                        shuffle=(sampler is None),
                        sampler=sampler,
                        batch_size = args.batch_size,
                        pin_memory=True)


    for epoch in range(0, 1):
        sampler.set_epoch(epoch)
        for data, label, ind in loader:
            data = data.cuda(args.gpu)
            label = label.cuda(args.gpu)
            ind = ind.cuda(args.gpu)

            print(f'D: {data},  {data.device}')
            print(f'I: {ind} {ind.device}',)
            print(f'L: {label} {label.device}')

            # take a random index to retrieve from the dataset

            rand_ind = torch.randint(300, (2,)).cuda(args.gpu)
            print('###############################')
            print(f'Random Index to Retrieve: {rand_ind}')
            print('###############################')
            datas, labels,inds = loader.dataset.get_sample(rand_ind)

            print(f'DS: {datas},  {data.device}')
            print(f'IS: {inds} {ind.device}')
            print(f'LS: {labels} {label.device}')
            print('========================================')


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='PyTorch ImageNet Training')
    parser.add_argument('--world-size', default=1, type=int,
                        help='number of nodes for distributed training')
    parser.add_argument('--rank', default=0, type=int,
                        help='node rank for distributed training')
    parser.add_argument('--dist-url', default='tcp://127.0.0.1:50000', type=str,
                        help='url used to set up distributed training')
    parser.add_argument('--dist-backend', default='nccl', type=str,
                        help='distributed backend')
    parser.add_argument('--gpu', default=None, type=int,
                        help='GPU id to use.')
    parser.add_argument('-b', '--batch-size', default=16, type=int,
                       metavar='N',
                       help='mini-batch size (default: 256), this is the total '
                            'batch size of all GPUs on the current node when '
                            'using Data Parallel or Distributed Data Parallel')
    parser.add_argument('-j', '--workers', default=16, type=int, metavar='N',
                        help='number of data loading workers (default: 4)')

    args, unparsed = parser.parse_known_args()

    arg = vars(args)
    for item in arg:
        print('{:.<30s}{:<50s}'.format(str(item), str(arg[item])))

    print('-------------------------------------------------------------------------------------------------')
    main(device=args.gpu, args=args)