I’m getting this issue when I am trying to map-tokenize a large custom data set. Looks like a multiprocessing issue. Running it with one proc or with a smaller set it seems work. I’ve tried different batch_size and still get the same errors. I also tried sharding it into smaller data sets, but that didn’t help. Thoughts? Thanks!
dataset[‘test’].map(lambda e: tokenizer(e[‘texts’]), batched = True, batch_size = 1000, num_proc  = 8)

error                                     Traceback (most recent call last)
 in 
----> 1 dataset[‘test’].map(lambda e: tokenizer(e[‘texts’]), batched = True, batch_size = 1000, num_proc  = 8)
/home/venv/lib/python3.6/site-packages/datasets/arrow_dataset.py in map(self, function, with_indices, input_columns, batched, batch_size, drop_last_batch, remove_columns, keep_in_memory, load_from_cache_file, cache_file_name, writer_batch_size, features, disable_nullable, fn_kwargs, num_proc, suffix_template, new_fingerprint)
1316                 logger.info(“Spawning {} processes”.format(num_proc))
1317                 results = [pool.apply_async(self.class._map_single, kwds=kwds) for kwds in kwds_per_shard]
 → 1318                 transformed_shards = [r.get() for r in results]
1319                 logger.info(“Concatenating {} shards from multiprocessing”.format(num_proc))
1320                 result = concatenate_datasets(transformed_shards)
/home/venv/lib/python3.6/site-packages/datasets/arrow_dataset.py in (.0)
1316                 logger.info(“Spawning {} processes”.format(num_proc))
1317                 results = [pool.apply_async(self.class._map_single, kwds=kwds) for kwds in kwds_per_shard]
 → 1318                 transformed_shards = [r.get() for r in results]
1319                 logger.info(“Concatenating {} shards from multiprocessing”.format(num_proc))
1320                 result = concatenate_datasets(transformed_shards)
/home/venv/lib/python3.6/site-packages/multiprocess/pool.py in get(self, timeout)
642             return self._value
643         else:
 → 644             raise self._value
645
646     def _set(self, i, obj):
/home/venv/lib/python3.6/site-packages/multiprocess/pool.py in _handle_tasks(taskqueue, put, outqueue, pool, cache)
422                         break
423                     try:
 → 424                         put(task)
425                     except Exception as e:
426                         job, idx = task[:2]
/home/venv/lib/python3.6/site-packages/multiprocess/connection.py in send(self, obj)
207         self._check_closed()
208         self._check_writable()
 → 209         self._send_bytes(_ForkingPickler.dumps(obj))
210
211     def recv_bytes(self, maxlength=None):
/home/venv/lib/python3.6/site-packages/multiprocess/connection.py in _send_bytes(self, buf)
394         n = len(buf)
395         # For wire compatibility with 3.2 and lower
 → 396         header = struct.pack("!i", n)
397         if n > 16384:
398             # The payload is large so Nagle’s algorithm won’t be triggered
error: ‘i’ format requires -2147483648 <= number <= 2147483647