pretrain

scripts/pretrain_base_model_0.yaml

@@ -1,11 +1,11 @@
 # The name of the model to pretrain. Choose from names in ``litgpt.config``. Mutually exclusive with
 # ``model_config``. (type: Optional[str], default: null)
-model_name: 'tangled-alpha-0.12-core'
+model_name: 'tangled-alpha-0.12-base'
 
 # A ``litgpt.Config`` object to define the model architecture. Mutually exclusive with
 # ``model_config``. (type: Optional[Config], default: null)
 model_config:
-  name: 'tangled-alpha-0.12-core'
+  name: 'tangled-alpha-0.12-base'
   block_size: 131072
   vocab_size: 65536
   padded_vocab_size: 65536
@@ -25,7 +25,7 @@ model_config:
 
 # Directory in which to save checkpoints and logs. If running in a Lightning Studio Job, look for it in
 # /teamspace/jobs/<job-name>/share. (type: <class 'Path'>, default: out/pretrain)
-out_dir: "../out/pretrain-core-0/"
+out_dir: "../out/pretrain-base-0/"
 
 # The precision to use for pretraining. Possible choices: "bf16-true", "bf16-mixed", "32-true". (type: Optional[str], default: null)
 # precision: bf16-mixed
@@ -61,7 +61,7 @@ train:
   global_batch_size: 512
 
   # Number of samples per data-parallel rank (type: int, default: 4)
-  micro_batch_size: 1
+  micro_batch_size: 2
 
   # Number of iterations with learning rate warmup active (type: int, default: 2000)
   lr_warmup_steps: 2000