Hugging Face's logo

craffel
/
comma-loss-test

Model card Files Community
craffel HF staff commited on
Commit
72d5ec2
·
1 Parent(s): 2905059

Add per-source models

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. arxiv_papers/checkpoint_metadata.json +9 -0
  2. arxiv_papers/config.yaml +144 -0
  3. arxiv_papers/model/decoder/0/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  4. arxiv_papers/model/decoder/0/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  5. arxiv_papers/model/decoder/0/pp_block/input_layernorm/model_weight.safetensors +3 -0
  6. arxiv_papers/model/decoder/0/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  7. arxiv_papers/model/decoder/0/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  8. arxiv_papers/model/decoder/0/pp_block/post_attention_layernorm/model_weight.safetensors +3 -0
  9. arxiv_papers/model/decoder/1/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  10. arxiv_papers/model/decoder/1/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  11. arxiv_papers/model/decoder/1/pp_block/input_layernorm/model_weight.safetensors +3 -0
  12. arxiv_papers/model/decoder/1/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  13. arxiv_papers/model/decoder/1/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  14. arxiv_papers/model/decoder/1/pp_block/post_attention_layernorm/model_weight.safetensors +3 -0
  15. arxiv_papers/model/decoder/10/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  16. arxiv_papers/model/decoder/10/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  17. arxiv_papers/model/decoder/10/pp_block/input_layernorm/model_weight.safetensors +3 -0
  18. arxiv_papers/model/decoder/10/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  19. arxiv_papers/model/decoder/10/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  20. arxiv_papers/model/decoder/10/pp_block/post_attention_layernorm/model_weight.safetensors +3 -0
  21. arxiv_papers/model/decoder/11/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  22. arxiv_papers/model/decoder/11/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  23. arxiv_papers/model/decoder/11/pp_block/input_layernorm/model_weight.safetensors +3 -0
  24. arxiv_papers/model/decoder/11/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  25. arxiv_papers/model/decoder/11/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  26. arxiv_papers/model/decoder/11/pp_block/post_attention_layernorm/model_weight.safetensors +3 -0
  27. arxiv_papers/model/decoder/12/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  28. arxiv_papers/model/decoder/12/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  29. arxiv_papers/model/decoder/12/pp_block/input_layernorm/model_weight.safetensors +3 -0
  30. arxiv_papers/model/decoder/12/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  31. arxiv_papers/model/decoder/12/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  32. arxiv_papers/model/decoder/12/pp_block/post_attention_layernorm/model_weight.safetensors +3 -0
  33. arxiv_papers/model/decoder/13/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  34. arxiv_papers/model/decoder/13/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  35. arxiv_papers/model/decoder/13/pp_block/input_layernorm/model_weight.safetensors +3 -0
  36. arxiv_papers/model/decoder/13/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  37. arxiv_papers/model/decoder/13/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  38. arxiv_papers/model/decoder/13/pp_block/post_attention_layernorm/model_weight.safetensors +3 -0
  39. arxiv_papers/model/decoder/14/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  40. arxiv_papers/model/decoder/14/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  41. arxiv_papers/model/decoder/14/pp_block/input_layernorm/model_weight.safetensors +3 -0
  42. arxiv_papers/model/decoder/14/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  43. arxiv_papers/model/decoder/14/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  44. arxiv_papers/model/decoder/14/pp_block/post_attention_layernorm/model_weight.safetensors +3 -0
  45. arxiv_papers/model/decoder/15/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  46. arxiv_papers/model/decoder/15/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  47. arxiv_papers/model/decoder/15/pp_block/input_layernorm/model_weight.safetensors +3 -0
  48. arxiv_papers/model/decoder/15/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  49. arxiv_papers/model/decoder/15/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors +3 -0
  50. arxiv_papers/model/decoder/15/pp_block/post_attention_layernorm/model_weight.safetensors +3 -0
arxiv_papers/checkpoint_metadata.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dp": 64,
3
+ "metas": {
4
+ "consumed_train_samples": 14336000,
5
+ "last_train_step": 14000
6
+ },
7
+ "tp": 1,
8
+ "version": "1.2"
9
+ }
arxiv_papers/config.yaml ADDED
@@ -0,0 +1,144 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ checkpoints:
2
+ checkpoint_interval: 500
3
+ checkpoints_path: /scratch/craffel/checkpoints/commav0p1-ablations-1p82G-commonpile0p1filteredarxiv_papers-seed-6-
4
+ checkpoints_path_is_shared_file_system: false
5
+ resume_checkpoint_path: s3://comma-v0.1-ablations/checkpoints/commav0p1-ablations-1p82G-commonpile0p1filteredarxiv_papers-seed-6-
6
+ save_initial_state: true
7
+ data:
8
+ dataset:
9
+ dataloader_type: single
10
+ dataset_max_tokens: null
11
+ dataset_weights: null
12
+ datasets:
13
+ - bits_per_token: 16
14
+ filename_pattern: .*\.ds$
15
+ folder: /scratch/dataset/commav0p1-ablations-1p82G-commonpile0p1filteredarxiv_papers-seed-6-/
16
+ original_folder: null
17
+ seed: 6
18
+ shuffle: true
19
+ skip_tokens: 0
20
+ pad_samples_to_global_batch_size: false
21
+ skip_in_stream: true
22
+ num_loading_workers: 0
23
+ seed: 6
24
+ experiment_logger:
25
+ tensorboard_logger:
26
+ push_to_hub_interval: 300
27
+ repo_id: craffel/commav0p1-ablations
28
+ repo_public: false
29
+ tensorboard_dir: /scratch/craffel/tensorboard-craffel-commav0p1-ablations
30
+ wandb_logger: null
31
+ general:
32
+ benchmark_csv_path: null
33
+ consumed_train_samples: 14336000
34
+ ignore_sanity_checks: true
35
+ project: commav0p1-ablations
36
+ run: commav0p1-ablations-1p82G-commonpile0p1filteredarxiv_papers-seed-6-
37
+ seed: 42
38
+ step: 14000
39
+ kill_switch_path: null
40
+ lighteval:
41
+ batch_size: 16
42
+ checkpoints_path: null
43
+ generation: null
44
+ logging:
45
+ hub_repo_details: null
46
+ hub_repo_results: null
47
+ hub_repo_tensorboard: craffel/commav0p1-ablations
48
+ local_output_path: /scratch/craffel/lighteval/commav0p1-ablations-1p82G-commonpile0p1filteredarxiv_papers-seed-6-
49
+ push_details_to_hub: false
50
+ push_results_to_hub: false
51
+ push_results_to_tensorboard: true
52
+ tensorboard_metric_prefix: e
53
+ parallelism:
54
+ dp: 8
55
+ expert_parallel_size: 1
56
+ pp: 1
57
+ pp_engine: 1f1b
58
+ tp: 1
59
+ tp_linear_async_communication: false
60
+ tp_mode: ALL_REDUCE
61
+ slurm_script_dir: /fsx/craffel/train/eval-scripts
62
+ slurm_template: /fsx/craffel/run_eval.slurm.jinja
63
+ tasks:
64
+ custom_tasks: brrr.lighteval.evaluation_tasks
65
+ dataset_loading_processes: 8
66
+ max_samples: 1000
67
+ multichoice_continuations_start_space: null
68
+ no_multichoice_continuations_start_space: null
69
+ num_fewshot_seeds: null
70
+ tasks: early-signal
71
+ wandb: null
72
+ logging:
73
+ iteration_step_info_interval: 1
74
+ log_level: info
75
+ log_level_replica: info
76
+ model:
77
+ ddp_bucket_cap_mb: 25
78
+ dtype: bfloat16
79
+ init_method:
80
+ std: 0.02
81
+ make_vocab_size_divisible_by: 1
82
+ model_config:
83
+ bos_token_id: 1
84
+ eos_token_id: 2
85
+ hidden_act: silu
86
+ hidden_size: 2048
87
+ initializer_range: 0.02
88
+ intermediate_size: 8192
89
+ is_llama_config: true
90
+ max_position_embeddings: 2048
91
+ num_attention_heads: 32
92
+ num_hidden_layers: 24
93
+ num_key_value_heads: 32
94
+ pad_token_id: null
95
+ pretraining_tp: 1
96
+ rms_norm_eps: 1.0e-05
97
+ rope_scaling: null
98
+ tie_word_embeddings: true
99
+ use_cache: true
100
+ vocab_size: 50272
101
+ optimizer:
102
+ accumulate_grad_in_fp32: true
103
+ adam_beta1: 0.9
104
+ adam_beta2: 0.95
105
+ adam_eps: 1.0e-08
106
+ clip_grad: 1.0
107
+ learning_rate_scheduler:
108
+ learning_rate: 0.0003
109
+ lr_decay_starting_step: null
110
+ lr_decay_steps: null
111
+ lr_decay_style: cosine
112
+ lr_warmup_steps: 500
113
+ lr_warmup_style: linear
114
+ min_decay_lr: 3.0e-05
115
+ torch_adam_is_fused: true
116
+ weight_decay: 0.1
117
+ zero_stage: 0
118
+ parallelism:
119
+ dp: 64
120
+ expert_parallel_size: 1
121
+ pp: 1
122
+ pp_engine: 1f1b
123
+ tp: 1
124
+ tp_linear_async_communication: true
125
+ tp_mode: REDUCE_SCATTER
126
+ profiler: null
127
+ s3_upload:
128
+ remove_after_upload: true
129
+ s5cmd_concurrency: 5
130
+ s5cmd_numworkers: 16
131
+ s5cmd_path: /fsx/craffel/miniconda3/envs/exp/bin/s5cmd
132
+ upload_s3_path: s3://comma-v0.1-ablations/checkpoints/commav0p1-ablations-1p82G-commonpile0p1filteredarxiv_papers-seed-6-
133
+ tokenizer:
134
+ tokenizer_max_length: null
135
+ tokenizer_name_or_path: gpt2
136
+ tokenizer_revision: null
137
+ tokens:
138
+ batch_accumulation_per_replica: 4
139
+ limit_test_batches: 0
140
+ limit_val_batches: 0
141
+ micro_batch_size: 4
142
+ sequence_length: 2048
143
+ train_steps: 14305
144
+ val_check_interval: 100
arxiv_papers/model/decoder/0/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:67d83ef891ea48ba66573e4cce8802a221d311b359e006af069bec3501a0b0d8
3
+ size 8388848
arxiv_papers/model/decoder/0/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:36c8af6c6a9859e869b48c7ec07b1ffb31d73bb4f51de223045ce811d14533cb
3
+ size 25166176
arxiv_papers/model/decoder/0/pp_block/input_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:93bcc323404a0d28579d429f2c897cf857daa05a1536d9df5b48f14edc2217b8
3
+ size 4192
arxiv_papers/model/decoder/0/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c96957e1959d1814f779e6298a5bbcaa39a3674e79e19ce9211dd81d632c427a
3
+ size 33554672
arxiv_papers/model/decoder/0/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8237f0e476c2342e14dbf6b7bb05ae7a1a584a976e3ecdef25a57b730b939ab0
3
+ size 67109160
arxiv_papers/model/decoder/0/pp_block/post_attention_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:055a7da36b180de0cd80f0c32bda26381173573d645ecbfda384776b55e68c21
3
+ size 4192
arxiv_papers/model/decoder/1/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7d8189ed7ae25d2471a6f04e2a0466c8bf0773bf453561ab3afdc280c3a8144d
3
+ size 8388848
arxiv_papers/model/decoder/1/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:980917c055e9681e06ea4dd249da917e28a0336a6cedd9f8b0747268a9d133a2
3
+ size 25166176
arxiv_papers/model/decoder/1/pp_block/input_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4174ab76a3a81521f2962558e01c4a7535f1d48fdcbdbc325a96200c066dab47
3
+ size 4192
arxiv_papers/model/decoder/1/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:47496c4f91bba54327b3fb27274701ecb91966f13986bb1e27927c2b6cf2e483
3
+ size 33554672
arxiv_papers/model/decoder/1/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5f6264a3e6dc9c6f05b47a37fd4297fb2cb0720bf392e4d58231831d48297579
3
+ size 67109160
arxiv_papers/model/decoder/1/pp_block/post_attention_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f0333fcc4da7a02bf3ed1b8a52601b3798ea2df0ca8a58fba1f93554ceed0c00
3
+ size 4192
arxiv_papers/model/decoder/10/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:97d2d7df20778b009164674f76b6178b461c328d4ed04bede15f35b742d9d797
3
+ size 8388848
arxiv_papers/model/decoder/10/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:05898d3f7854f9b466f2ab75b580de88a50202636622097645d3e667fb4a45f9
3
+ size 25166176
arxiv_papers/model/decoder/10/pp_block/input_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bd7dcd44fab01b7aad5a21a1ac4c28553c91ac3ae360aadb01a4c284ac7f6c0a
3
+ size 4192
arxiv_papers/model/decoder/10/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:09b03b0322b643c697edc1dbd0b020190264f2614439e165b9e784507a94c8ee
3
+ size 33554672
arxiv_papers/model/decoder/10/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:290f71feb11dd01b57e170e75d594067f40cf59746b22e819e56bab2738df394
3
+ size 67109160
arxiv_papers/model/decoder/10/pp_block/post_attention_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dcaa60a3b03cbf1f8fc9c78429a4f3d4455021198a1074cac73d6a55e5778ba3
3
+ size 4192
arxiv_papers/model/decoder/11/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c1cd9e9276953c817668c9cb64a60b51e905cb8af7a993e999d7e93c59d64a66
3
+ size 8388848
arxiv_papers/model/decoder/11/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5d226ec81c772785886795c1659d4685b0fc6058e29107945a576123329741e9
3
+ size 25166176
arxiv_papers/model/decoder/11/pp_block/input_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:628daa47fb0caec0dd0b3eb435815af7315bc66a846ad6892eddb847fc0f725e
3
+ size 4192
arxiv_papers/model/decoder/11/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:02a3ce9143b1c9765c4c5f9dffc641de6a5213f6bdffebfeeb31e42a22475c97
3
+ size 33554672
arxiv_papers/model/decoder/11/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f1593d722651fd5b567fd6f87d69bafe6f1c4f635f0c48a156fd13859f17a26d
3
+ size 67109160
arxiv_papers/model/decoder/11/pp_block/post_attention_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3effe1931b95d6390a570c5840bd4f039157436aa6f59e1a67110d01e1ff0421
3
+ size 4192
arxiv_papers/model/decoder/12/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0821a089c8e422d4780c7416103c3c8a800c6ec4a558e779e0dedc7e19a94d05
3
+ size 8388848
arxiv_papers/model/decoder/12/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9afd4705e7ffea40f53d24932d688f06a7534f3b386145e076f732330030335f
3
+ size 25166176
arxiv_papers/model/decoder/12/pp_block/input_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7e0a399ae2c62944fb5bf4e8907ac674674df7bae5b2a264ab23ddf252f4911c
3
+ size 4192
arxiv_papers/model/decoder/12/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4c2fcdd0bcf96aaa2a7e55472ab10488acff6fb6768e619473eb9c7e71f185cc
3
+ size 33554672
arxiv_papers/model/decoder/12/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:47548fd48a98477a3b1a15e76cbed1a9041c03a3dada253737df4cc07c38f168
3
+ size 67109160
arxiv_papers/model/decoder/12/pp_block/post_attention_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d4702911bc67577212f2b0911c95fff717d6175a2671451145a960c818980c81
3
+ size 4192
arxiv_papers/model/decoder/13/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cfd448ae1c26abd57462009715fc979792ad50e62ad7fcb103f6ae5bdfba5092
3
+ size 8388848
arxiv_papers/model/decoder/13/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fa1368596dec823114975b8e29361dc04a283b3827a294ddbcc59b6cd225a513
3
+ size 25166176
arxiv_papers/model/decoder/13/pp_block/input_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:821695b511e08007a28eeb5aaa3b7eff6aee838ca9e054f5255f662371d2a219
3
+ size 4192
arxiv_papers/model/decoder/13/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3aa1e03deac41391e87bf6280b7004761ab9d0899be016bbe0902a235af6eaa4
3
+ size 33554672
arxiv_papers/model/decoder/13/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:863e26bfeafc90e78150f561b6a8acf40f3eb19d7118bb5379c81559db57f8e0
3
+ size 67109160
arxiv_papers/model/decoder/13/pp_block/post_attention_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bd1f6f0b0b7b5345d7ba8cc6d793ec937017b2497a3d98a4a7c519cc35c1153a
3
+ size 4192
arxiv_papers/model/decoder/14/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b26a00c3825186b3633e7f7d06a003a681b89bf2b81c355f35120a80ed33d437
3
+ size 8388848
arxiv_papers/model/decoder/14/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:157182612ee0443a3caa372af097c5c01b80c83927762e6ded3b582ca4bf1e85
3
+ size 25166176
arxiv_papers/model/decoder/14/pp_block/input_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:01c43197a5aa2648b8f617193ea635ae1bab34270deac2c904b5a66025a4f3ee
3
+ size 4192
arxiv_papers/model/decoder/14/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:93f55762482682435ca29c270f4807610b03aca418d4a8fb444b5bbf315f8be1
3
+ size 33554672
arxiv_papers/model/decoder/14/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6756a2c55358de350fb1a712a55a4d7850a7a76567532cb38b5fc5cb32e392e3
3
+ size 67109160
arxiv_papers/model/decoder/14/pp_block/post_attention_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a41e1bdf54a7f7d8016e4756abc5676b1cbfc3a2efa3cfc6f997a9ecdccb6ab5
3
+ size 4192
arxiv_papers/model/decoder/15/pp_block/attn/o_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:686dcf4083072273359f92e637549e458c26ef8d288b48d5e3e6d3ff10df4121
3
+ size 8388848
arxiv_papers/model/decoder/15/pp_block/attn/qkv_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bd6ecb056a320eb7b5e6473fa82c3eea181cdead46b6c80559bce7646216186b
3
+ size 25166176
arxiv_papers/model/decoder/15/pp_block/input_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8bc323f62a8242a15e10dbb8d27d73abad6e372a63a13af05f0dfec59476f456
3
+ size 4192
arxiv_papers/model/decoder/15/pp_block/mlp/down_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6b5d78fa8596785210f87d55723faeb0ab46e7dd21deb9f8e1977611a681ad0f
3
+ size 33554672
arxiv_papers/model/decoder/15/pp_block/mlp/gate_up_proj/model_weight_pp-rank-0-of-1_tp-rank-0-of-1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3d96f01c1527e7f4d5f27b4bab6a4dd85a708aaf003527ba73ec5a23a042c507
3
+ size 67109160
arxiv_papers/model/decoder/15/pp_block/post_attention_layernorm/model_weight.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e5552c575a432bf1f33e7fb34f2ea9da1a95d39b2c717422ec9699e9108d9324
3
+ size 4192