Merge branch 'main' of hf.co:tangledgroup/tangled-alpha-0.10-core

README.md

@@ -108,6 +108,16 @@ Epoch 1 | iter 512 step 8 | loss train: 11.973, val: n/a | iter time: 403.80 ms
 Epoch 1 | iter 576 step 9 | loss train: 11.972, val: n/a | iter time: 403.23 ms (step) remaining time: 6 days, 15:21:59
 Epoch 1 | iter 640 step 10 | loss train: 11.967, val: n/a | iter time: 403.38 ms (step) remaining time: 6 days, 13:43:53
 # ...
+Epoch 2 | iter 1364224 step 21316 | loss train: 2.805, val: 2.809 | iter time: 404.72 ms (step) remaining time: 0:00:06
+Validating ...
+Final evaluation | val loss: 2.809 | val ppl: 16.592
+Saving checkpoint to '../out/pretrain-core-0/final/lit_model.pth'
+----------------------------------------
+| Performance
+| - Total tokens  : 11,186,768,000
+| - Training Time : 53900.17 s
+| - Tok/sec       : 34385052.80 tok/s
+| ----------------------------------------
 ```
 
 Backup `wandb`:
@@ -133,4 +143,57 @@ CUDA_VISIBLE_DEVICES=0 CUDA_LAUNCH_BLOCKING=0 PYTORCH_CUDA_ALLOC_CONF=expandable
 ```
 
 ```
+Tasks                           |Version|Filter|n-shot|        Metric         |   |Value |   |Stderr|
+|-----------------------------------------------------------|-------|------|-----:|-----------------------|---|-----:|---|------|
+|leaderboard                                                |    N/A|      |      |                       |   |      |   |      |
+| - leaderboard_bbh                                         |    N/A|      |      |                       |   |      |   |      |
+|  - leaderboard_bbh_boolean_expressions                    |      1|none  |     3|acc_norm               |↑  |0.4680|±  |0.0316|
+|  - leaderboard_bbh_causal_judgement                       |      1|none  |     3|acc_norm               |↑  |0.5187|±  |0.0366|
+|  - leaderboard_bbh_date_understanding                     |      1|none  |     3|acc_norm               |↑  |0.2080|±  |0.0257|
+|  - leaderboard_bbh_disambiguation_qa                      |      1|none  |     3|acc_norm               |↑  |0.3760|±  |0.0307|
+|  - leaderboard_bbh_formal_fallacies                       |      1|none  |     3|acc_norm               |↑  |0.5320|±  |0.0316|
+|  - leaderboard_bbh_geometric_shapes                       |      1|none  |     3|acc_norm               |↑  |0.1160|±  |0.0203|
+|  - leaderboard_bbh_hyperbaton                             |      1|none  |     3|acc_norm               |↑  |0.5160|±  |0.0317|
+|  - leaderboard_bbh_logical_deduction_five_objects         |      1|none  |     3|acc_norm               |↑  |0.2000|±  |0.0253|
+|  - leaderboard_bbh_logical_deduction_seven_objects        |      1|none  |     3|acc_norm               |↑  |0.1280|±  |0.0212|
+|  - leaderboard_bbh_logical_deduction_three_objects        |      1|none  |     3|acc_norm               |↑  |0.3440|±  |0.0301|
+|  - leaderboard_bbh_movie_recommendation                   |      1|none  |     3|acc_norm               |↑  |0.2400|±  |0.0271|
+|  - leaderboard_bbh_navigate                               |      1|none  |     3|acc_norm               |↑  |0.4200|±  |0.0313|
+|  - leaderboard_bbh_object_counting                        |      1|none  |     3|acc_norm               |↑  |0.0560|±  |0.0146|
+|  - leaderboard_bbh_penguins_in_a_table                    |      1|none  |     3|acc_norm               |↑  |0.2260|±  |0.0347|
+|  - leaderboard_bbh_reasoning_about_colored_objects        |      1|none  |     3|acc_norm               |↑  |0.1520|±  |0.0228|
+|  - leaderboard_bbh_ruin_names                             |      1|none  |     3|acc_norm               |↑  |0.2080|±  |0.0257|
+|  - leaderboard_bbh_salient_translation_error_detection    |      1|none  |     3|acc_norm               |↑  |0.2240|±  |0.0264|
+|  - leaderboard_bbh_snarks                                 |      1|none  |     3|acc_norm               |↑  |0.4831|±  |0.0376|
+|  - leaderboard_bbh_sports_understanding                   |      1|none  |     3|acc_norm               |↑  |0.4640|±  |0.0316|
+|  - leaderboard_bbh_temporal_sequences                     |      1|none  |     3|acc_norm               |↑  |0.2520|±  |0.0275|
+|  - leaderboard_bbh_tracking_shuffled_objects_five_objects |      1|none  |     3|acc_norm               |↑  |0.1720|±  |0.0239|
+|  - leaderboard_bbh_tracking_shuffled_objects_seven_objects|      1|none  |     3|acc_norm               |↑  |0.1480|±  |0.0225|
+|  - leaderboard_bbh_tracking_shuffled_objects_three_objects|      1|none  |     3|acc_norm               |↑  |0.3320|±  |0.0298|
+|  - leaderboard_bbh_web_of_lies                            |      1|none  |     3|acc_norm               |↑  |0.4880|±  |0.0317|
+| - leaderboard_gpqa                                        |    N/A|      |      |                       |   |      |   |      |
+|  - leaderboard_gpqa_diamond                               |      1|none  |     0|acc_norm               |↑  |0.2071|±  |0.0289|
+|  - leaderboard_gpqa_extended                              |      1|none  |     0|acc_norm               |↑  |0.2619|±  |0.0188|
+|  - leaderboard_gpqa_main                                  |      1|none  |     0|acc_norm               |↑  |0.2545|±  |0.0206|
+| - leaderboard_ifeval                                      |      3|none  |     0|inst_level_loose_acc   |↑  |0.2710|±  |   N/A|
+|                                                           |       |none  |     0|inst_level_strict_acc  |↑  |0.2626|±  |   N/A|
+|                                                           |       |none  |     0|prompt_level_loose_acc |↑  |0.1165|±  |0.0138|
+|                                                           |       |none  |     0|prompt_level_strict_acc|↑  |0.1128|±  |0.0136|
+| - leaderboard_math_hard                                   |    N/A|      |      |                       |   |      |   |      |
+|  - leaderboard_math_algebra_hard                          |      2|none  |     4|exact_match            |↑  |0.0194|±  |0.0040|
+|  - leaderboard_math_counting_and_prob_hard                |      2|none  |     4|exact_match            |↑  |0.0148|±  |0.0055|
+|  - leaderboard_math_geometry_hard                         |      2|none  |     4|exact_match            |↑  |0.0042|±  |0.0029|
+|  - leaderboard_math_intermediate_algebra_hard             |      2|none  |     4|exact_match            |↑  |0.0111|±  |0.0035|
+|  - leaderboard_math_num_theory_hard                       |      2|none  |     4|exact_match            |↑  |0.0056|±  |0.0032|
+|  - leaderboard_math_prealgebra_hard                       |      2|none  |     4|exact_match            |↑  |0.0161|±  |0.0043|
+|  - leaderboard_math_precalculus_hard                      |      2|none  |     4|exact_match            |↑  |0.0092|±  |0.0041|
+| - leaderboard_mmlu_pro                                    |    0.1|none  |     5|acc                    |↑  |0.1184|±  |0.0029|
+| - leaderboard_musr                                        |    N/A|      |      |                       |   |      |   |      |
+|  - leaderboard_musr_murder_mysteries                      |      1|none  |     0|acc_norm               |↑  |0.5240|±  |0.0316|
+|  - leaderboard_musr_object_placements                     |      1|none  |     0|acc_norm               |↑  |0.2344|±  |0.0265|
+|  - leaderboard_musr_team_allocation                       |      1|none  |     0|acc_norm               |↑  |0.3000|±  |0.0290|
+```
+
+```bash
+litgpt convert_pretrained_checkpoint ../out/pretrain-core-0/final ../out/pretrain-core-0/checkpoint
 ```

config-1.json

@@ -0,0 +1,29 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "eos_token_id": 1,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 2048,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 4,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 16000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.45.0.dev0",
+  "use_cache": true,
+  "vocab_size": 131072
+}

scripts/pretrain_core_model_1.yaml

@@ -0,0 +1,154 @@
+# The name of the model to pretrain. Choose from names in ``litgpt.config``. Mutually exclusive with
+# ``model_config``. (type: Optional[str], default: null)
+model_name: 'tangled-alpha-0.10-core'
+
+# A ``litgpt.Config`` object to define the model architecture. Mutually exclusive with
+# ``model_config``. (type: Optional[Config], default: null)
+model_config:
+  name: 'tangled-alpha-0.10-core'
+  block_size: 131072
+  vocab_size: 131072
+  padded_vocab_size: 131072
+  n_layer: 32
+  n_head: 12
+  n_embd: 768
+  n_query_groups: 4
+  rotary_percentage: 1.0
+  parallel_residual: False
+  bias: False
+  norm_class_name: "RMSNorm"
+  mlp_class_name: "LLaMAMLP"
+  intermediate_size: 2048 # n_embd * 2.666
+  norm_eps: 1e-5
+  rope_base: 16000 # https://arxiv.org/pdf/2405.14591
+  head_size: 64 # n_embd / n_head
+
+# Directory in which to save checkpoints and logs. If running in a Lightning Studio Job, look for it in
+# /teamspace/jobs/<job-name>/share. (type: <class 'Path'>, default: out/pretrain)
+out_dir: "../out/pretrain-core-1/"
+
+# The precision to use for pretraining. Possible choices: "bf16-true", "bf16-mixed", "32-true". (type: Optional[str], default: null)
+# precision: bf16-mixed
+precision: bf16-true
+
+# Optional path to a checkpoint directory to initialize the model from.
+# Useful for continued pretraining. Mutually exclusive with ``resume``. (type: Optional[Path], default: null)
+initial_checkpoint_dir: "../out/pretrain-core-0/checkpoint"
+
+# Path to a checkpoint directory to resume from in case training was interrupted, or ``True`` to resume
+# from the latest checkpoint in ``out_dir``. An error will be raised if no checkpoint is found. Passing
+# ``'auto'`` will resume from the latest checkpoint but not error if no checkpoint exists.
+# (type: Union[bool, Literal["auto"], Path], default: False)
+resume:
+
+# Data-related arguments. If not provided, the default is ``litgpt.data.TinyLlama``.
+data:
+  class_path: LitData
+
+  init_args:
+    data_path: "../core-data-1-1025-2049-2049-8000/"
+    num_workers: 32
+
+# Training-related arguments. See ``litgpt.args.TrainArgs`` for details
+train:
+  # Number of optimizer steps between saving checkpoints (type: Optional[int], default: 1000)
+  save_interval: 50
+
+  # Number of iterations between logging calls (type: int, default: 1)
+  log_interval: 1
+
+  # Number of samples between optimizer steps across data-parallel ranks (type: int, default: 512)
+  global_batch_size: 512
+
+  # Number of samples per data-parallel rank (type: int, default: 4)
+  micro_batch_size: 4
+
+  # Number of iterations with learning rate warmup active (type: int, default: 2000)
+  lr_warmup_steps: 0
+
+  # Number of epochs to train on (type: Optional[int], default: null)
+  epochs:
+
+  # Total number of tokens to train on (type: Optional[int], default: 3000000000000)
+  max_tokens: 1830709785
+
+  # Limits the number of optimizer steps to run. (type: Optional[int], default: null)
+  max_steps:
+
+  # Limits the length of samples. Off by default (type: Optional[int], default: null)
+  max_seq_length: 2049
+
+  # Whether to tie the embedding weights with the language modeling head weights. (type: Optional[bool], default: False)
+  tie_embeddings: false
+
+  #   (type: Optional[float], default: 1.0)
+  max_norm: 1.0
+
+  #   (type: float, default: 4e-05)
+  min_lr: 1e-6
+
+# Evaluation-related arguments. See ``litgpt.args.EvalArgs`` for details
+eval:
+  # Number of optimizer steps between evaluation calls (type: int, default: 1000)
+  interval: 50
+
+  # Number of tokens to generate (type: Optional[int], default: null)
+  max_new_tokens:
+
+  # Number of iterations (type: int, default: 100)
+  max_iters: 100
+
+  # Whether to evaluate on the validation set at the beginning of the training
+  initial_validation: true
+
+  # Whether to evaluate on the validation set at the end the training
+  final_validation: true
+
+# Optimizer-related arguments
+# optimizer:
+#   class_path: torch.optim.AdamW
+#   init_args:
+#     # (type: float, default: 0.001)
+#     lr: 3e-4
+#     # (type: float, default: 0.01)
+#     weight_decay: 0.01
+#     # (type: tuple, default: (0.9,0.999))
+#     betas:
+#       - 0.9
+#       - 0.999
+
+# optimizer:
+#   class_path: sophia_opt.SophiaG
+#   init_args:
+#     lr: 3e-4
+#     betas:
+#       - 0.9
+#       - 0.95
+#     rho: 0.05
+#     weight_decay: 0.1
+
+optimizer:
+  class_path: sophia_opt.SophiaG
+  init_args:
+    lr: 1e-5
+    betas:
+      - 0.965
+      - 0.99
+    rho: 0.04
+    weight_decay: 1e-1
+
+# How many devices/GPUs to use. Uses all GPUs by default. (type: Union[int, str], default: auto)
+devices: auto
+
+# How many nodes to use. (type: int, default: 1)
+num_nodes: 1
+
+# Optional path to the tokenizer dir that was used for preprocessing the dataset. Only some data
+# module require this. (type: Optional[Path], default: null)
+tokenizer_dir: "../tokenizer"
+
+# The name of the logger to send metrics to. (type: Literal['wandb', 'tensorboard', 'csv'], default: tensorboard)
+logger_name: "wandb"
+
+# The random seed to use for reproducibility. (type: int, default: 42)
+seed: 23