Hey,
Please have a look at the following example:
import numpy as np
import torch

from torch.utils.data import Dataset, DataLoader


class MyDataset(Dataset):
    def __init__(self):
        self.data = np.random.randint(0, 100, (10, 2, 2))
        self.max_id = np.ones(len(self.data), dtype=np.int16)


    def __len__(self):
        return len(self.data)


    def __getitem__(self, idx):
        self.max_id[idx] = np.max(self.data[idx, :, :])
        return self.data[idx, :, :]


def main():
    np.random.seed(42)
    torch.manual_seed(42)
    dataset = MyDataset()
    loader = DataLoader(dataset, batch_size=4, shuffle=True, num_workers=1)
    loader_iter = iter(loader)

    print(f'Initial State: {dataset.max_id}')

    batch = next(loader_iter)

    print(f'After num_worker=1: {dataset.max_id}')

    loader = DataLoader(dataset, batch_size=4, shuffle=True, num_workers=0)
    loader_iter = iter(loader)
    batch = next(loader_iter)

    print(f'After num_worker=0: {dataset.max_id}')


if __name__ == '__main__':
    main()

In the last case, where DataLoader(..., num_workers=0, ...), I observe the intended behaviour: dataset.max_id is updated.
How can I achieve this with e.g. DataLoader(..., num_workers=1, ...)?
I am using this in the context of large image files, where I want to read information from these images only upon calling them for training/validation. Thank you for any hint/advice!
Cheers