pretrain

scripts/pretrain_base_model_0.yaml

@@ -46,7 +46,7 @@ data:
   class_path: LitData
 
   init_args:
-    data_path: "../core-data-0-0-1073741824-1025-16000/"
+    data_path: "../base-data-0-0-1073741824-8193-2000/"
     num_workers: 32
 
 # Training-related arguments. See ``litgpt.args.TrainArgs`` for details
@@ -70,7 +70,7 @@ train:
   epochs:
 
   # Total number of tokens to train on (type: Optional[int], default: 3000000000000)
-  max_tokens: 11186775175
+  max_tokens: 32706456
 
   # Limits the number of optimizer steps to run. (type: Optional[int], default: null)
   max_steps: