don't add position_ids for evals (#591)
Browse files
src/axolotl/utils/trainer.py
CHANGED
|
@@ -403,10 +403,11 @@ def process_datasets_for_packing(cfg, train_dataset, eval_dataset):
|
|
| 403 |
|
| 404 |
if cfg.sample_packing:
|
| 405 |
train_dataset = train_dataset.map(add_position_ids, num_proc=os.cpu_count())
|
| 406 |
-
if
|
| 407 |
-
|
| 408 |
-
|
| 409 |
-
|
|
|
|
| 410 |
return train_dataset, eval_dataset
|
| 411 |
|
| 412 |
|
|
|
|
| 403 |
|
| 404 |
if cfg.sample_packing:
|
| 405 |
train_dataset = train_dataset.map(add_position_ids, num_proc=os.cpu_count())
|
| 406 |
+
if cfg.eval_sample_packing is not False:
|
| 407 |
+
if eval_dataset:
|
| 408 |
+
eval_dataset = eval_dataset.map(
|
| 409 |
+
add_position_ids, num_proc=os.cpu_count()
|
| 410 |
+
)
|
| 411 |
return train_dataset, eval_dataset
|
| 412 |
|
| 413 |
|