A simple example to split dataset among n_workers:
import torch
import torch.nn as nn
import torchvision
import torchvision.transforms as transforms
import numpy as np
import time
from worker import worker
from worker_new import worker_new
#the number of workers here:
n_workers = 4
testset = torchvision.datasets.CIFAR10(root=’./data’, train=False, download=True, transform=transforms.ToTensor())
#Split testset, you can access the data from worker 1 with Worker_data[1], and so on.
temp = tuple([len(testset)//n_workers for i in range(n_workers)])
Worker_data = torch.utils.data.random_split(testset, temp)