pretrain core
Browse files
scripts/pretrain_core_model_0.yaml
CHANGED
|
@@ -21,7 +21,7 @@ model_config:
|
|
| 21 |
intermediate_size: 2048 # n_embd * 2.666
|
| 22 |
norm_eps: 1e-5
|
| 23 |
rope_base: 4300 # https://arxiv.org/pdf/2405.14591
|
| 24 |
-
head_size:
|
| 25 |
|
| 26 |
# Directory in which to save checkpoints and logs. If running in a Lightning Studio Job, look for it in
|
| 27 |
# /teamspace/jobs/<job-name>/share. (type: <class 'Path'>, default: out/pretrain)
|
|
|
|
| 21 |
intermediate_size: 2048 # n_embd * 2.666
|
| 22 |
norm_eps: 1e-5
|
| 23 |
rope_base: 4300 # https://arxiv.org/pdf/2405.14591
|
| 24 |
+
head_size: 64 # n_embd / n_head
|
| 25 |
|
| 26 |
# Directory in which to save checkpoints and logs. If running in a Lightning Studio Job, look for it in
|
| 27 |
# /teamspace/jobs/<job-name>/share. (type: <class 'Path'>, default: out/pretrain)
|