I am also experiencing this issue, whether or not I set my num_workers to 0.
I have a 1TB dataset and here is my code:
class DicomDatasetRetriever(torch.utils.data.Dataset):
    def __init__(self, df, transforms=[], mix_ratio=1, mode='val'):
        self.df_main = df.copy()
        self.mode = mode
        self.mix_ratio = mix_ratio
        if self.mode == 'val':
            self.df = self.df_main
        else:
            self.update_train_df()
        self.lut = df[['SOPInstanceUID', 'image_path']].set_index('SOPInstanceUID')
        
        if not(len(transforms)):
            self.transforms = None
        else:
            self.transforms = A.Compose(transforms)
        
        self.default_transforms = A.Compose([
            A.Normalize(0.449, 0.226),
            ToTensorV2(),
        ])

    def __len__(self):
        return len(self.df)
    
    def __getitem__(self, idx):
        row = self.df.iloc[idx]
        study = row.StudyInstanceUID
        img_id = row.SOPInstanceUID
        
        img = self.load_image(img_id)
        pe_ratio = row.r_pe_present_on_image
        target = row.pe_present_on_image

        # transforms
        if self.transforms is not None and self.mode != 'val' and (row[3] == 1 or random.random() < 0.1):
            img = self.transforms(image=img)['image']
        
        # default transformation
        img = self.default_transforms(image=img)['image']

        return {
            'img': img,
            'img_id': img_id,
            'study_id': study,
            'pe_ratio': torch.tensor([pe_ratio]).float(),
            'target': torch.tensor([target]).float()
        }

    def load_image(self, img_id):
#         img = cv2.imread(self.lut.loc[img_id, 'image_path'])
        with open(self.lut.loc[img_id, 'image_path'], 'rb') as f:
            img = JPEG.decode(f.read())
        if img is None:
            print(f"Warning while trying to load image. No file at {file_path}")
            img = np.zeros(shape=SHAPE) 
        img = img.astype(np.float32)
        img /= 255
        return img
    
    def update_train_df(self):
        df0 = self.df_main[self.df_main.pe_present_on_image==0]
        df1 = self.df_main[self.df_main.pe_present_on_image==1]
        df0 = df0.sample(frac=1)
        upto = min(int(self.mix_ratio * len(df1)), len(df0))
        self.df = pd.concat([df0.iloc[:upto],df1], axis=0)
        self.df = self.df.sample(frac=1)

class DataLoaders:

    self.train_loader = torch.utils.data.DataLoader(
            self.train_dataset, batch_size=self.batch_size, shuffle=True,
            num_workers=self.num_workers, pin_memory=True)