Hi I have a custom dataset of structure:
DatasetDict({
train: Dataset({
    features: ['sentence', 'label'],
    num_rows: 12638343
})
test: Dataset({
    features: ['sentence', 'label'],
    num_rows: 1560290
})
validation: Dataset({
    features: ['sentence', 'label'],
    num_rows: 1404261
})
})

When I encode training example using map method from dataset with tokenizer defined:
Note I can only use num_proc = 1 no matter how I code if name == “main”: after defining tokenizer for multiprocess it will say “tokenizer” not found… anyway lets focus on this error for the time being
train_encodings = dataset["train"].map(lambda examples: tokenizer(examples['sentence'], truncation = True, padding = True), batched=True, num_proc = 1, keep_in_memory = True)

I ran into Index Error 10% through, no matter I change batch_size it will still stuck at this:
C:\Users\xxxxx\AppData\Roaming\Python\Python38\site-packages\datasets\table.py:84: 
RuntimeWarning: overflow encountered in int_scalars
k = i + ((j - i) * (x - arr[i]) // (arr[j] - arr[i]))
---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
<ipython-input-13-b3bd4b3dbd29> in <module>
  1 ## Pass our Tokenizer to our Train, test, validation set ##
  2 
----> 3 train_encodings = dataset["train"].map(lambda examples: tokenizer(examples['sentence'], 
truncation = True, padding = True), batched=True, num_proc = 1, keep_in_memory = True)
  4 
  5 # val_encodings = dataset["validation"].map(lambda examples: tokenizer(examples['sentence'], 
truncation = True, padding = True), batched=True, num_proc = 1, keep_in_memory = True)

~\AppData\Roaming\Python\Python38\site-packages\datasets\arrow_dataset.py in map(self, 
function, with_indices, input_columns, batched, batch_size, drop_last_batch, remove_columns, 
keep_in_memory, load_from_cache_file, cache_file_name, writer_batch_size, features, 
disable_nullable, fn_kwargs, num_proc, suffix_template, new_fingerprint)
1481 
1482         if num_proc is None or num_proc == 1:
-> 1483             return self._map_single(
1484                 function=function,
1485                 with_indices=with_indices,

~\AppData\Roaming\Python\Python38\site-packages\datasets\arrow_dataset.py in wrapper(*args, 
**kwargs)
172         }
173         # apply actual function
--> 174         out: Union["Dataset", "DatasetDict"] = func(self, *args, **kwargs)
175         datasets: List["Dataset"] = list(out.values()) if isinstance(out, dict) else [out]
176         # re-apply format to the output

~\AppData\Roaming\Python\Python38\site-packages\datasets\fingerprint.py in wrapper(*args, 
**kwargs)
338             # Call actual function
339 
--> 340             out = func(self, *args, **kwargs)
341 
342             # Update fingerprint of in-place transforms + update in-place history of transforms

~\AppData\Roaming\Python\Python38\site-packages\datasets\arrow_dataset.py in 
_map_single(self, function, with_indices, input_columns, batched, batch_size, drop_last_batch, 
remove_columns, keep_in_memory, load_from_cache_file, cache_file_name, writer_batch_size, 
features, disable_nullable, fn_kwargs, new_fingerprint, rank, offset)
1812                         if drop_last_batch and i + batch_size > input_dataset.num_rows:
1813                             continue
-> 1814                         batch = input_dataset[i : i + batch_size]
1815                         indices = list(
1816                             range(*(slice(i, i + batch_size).indices(input_dataset.num_rows)))

~\AppData\Roaming\Python\Python38\site-packages\datasets\arrow_dataset.py in __getitem__(self, 
key)
1343     def __getitem__(self, key: Union[int, slice, str]) -> Union[Dict, List]:
1344         """Can be used to index columns (by string names) or rows (by integer index or iterable of 
indices or bools)."""
-> 1345         return self._getitem(
1346             key,
1347             format_type=self._format_type,

~\AppData\Roaming\Python\Python38\site-packages\datasets\arrow_dataset.py in _getitem(self, 
key, format_type, format_columns, output_all_columns, format_kwargs)
1335         format_kwargs = format_kwargs if format_kwargs is not None else {}
1336         formatter = get_formatter(format_type, **format_kwargs)
-> 1337         pa_subtable = query_table(self._data, key, indices=self._indices if self._indices is not 
None else None)
1338         formatted_output = format_table(
1339             pa_subtable, key, formatter=formatter, format_columns=format_columns, 
output_all_columns=output_all_columns

~\AppData\Roaming\Python\Python38\site-packages\datasets\formatting\formatting.py in 
 query_table(table, key, indices)
 366     # Query the main table
 367     if indices is None:
--> 368         pa_subtable = _query_table(table, key)
 369     else:
 370         pa_subtable = _query_table_with_indices_mapping(table, key, indices=indices)

~\AppData\Roaming\Python\Python38\site-packages\datasets\formatting\formatting.py in 
_query_table(table, key)
 82     if isinstance(key, range):
 83         if _is_range_contiguous(key) and key.start >= 0:
---> 84             return table.fast_slice(key.start, key.stop - key.start)
 85         else:
 86             pass  # treat as an iterable

~\AppData\Roaming\Python\Python38\site-packages\datasets\table.py in fast_slice(self, offset, length)
131             batches[0] = batches[0].slice(offset - self._offsets[i])
132         else:
--> 133             j = _interpolation_search(self._offsets, offset + length - 1)
134             batches = self._batches[i : j + 1]
135             batches[-1] = batches[-1].slice(0, offset + length - self._offsets[j])

~\AppData\Roaming\Python\Python38\site-packages\datasets\table.py in _interpolation_search(arr, x)
 89         else:
 90             i, j = i, k
---> 91     raise IndexError(f"Invalid query '{x}' for size {arr[-1] if len(arr) else 'none'}.")
 92 
 93 

IndexError: Invalid query '1698999' for size 12638343.

I could have load the csv using pandas then encode on it but I close to run out of memory on that so I try to use datasets_load class… Any help would be appreciated! Thanks