I had the same issue when opening a tarfile. A quick fix is to open a zipfile handle once at the start of __getitem__.
class MyDataSet(Dataset):
   def __init__(self, filename):
        self.zip_handle = None
        self.fname = filename

    def __getitem__(self, x):
        if self.zip_handle is None:
            self.zip_handle = zipfile.ZipFile(self.fname, 'r')
        # do stuff