| experiment: | |
| seed: 42 | |
| name: gpt2_600k-1800000-05 | |
| group: childes-size-english | |
| dry_run: false | |
| offline_run: false | |
| evaluate_segmentation: true | |
| evaluate_babyslm: true | |
| blimp_tasks: null | |
| resume_checkpoint_path: null | |
| resume_run_id: null | |
| dataset: | |
| name: phonemetransformers/CHILDES | |
| subconfig: English | |
| text_column: phonemized_utterance | |
| is_phonemes: true | |
| max_age: 120 | |
| remove_child_utterances: true | |
| tokenizer: | |
| name: phonemetransformers/CHILDES-English-phoneme-tokenizer | |
| data_preprocessing: | |
| max_input_length: 128 | |
| join_utts: static | |
| remove_word_boundaries: true | |
| subsample: 1800000 | |
| subsample_type: tokens | |
| model: | |
| name: gpt2_lm | |
| model_kwargs: | |
| n_layer: 3 | |
| n_head: 4 | |
| n_embd: 128 | |
| n_positions: 256 | |
| n_inner: 512 | |
| resid_pdrop: 0.5 | |
| embd_pdrop: 0.5 | |
| attn_pdrop: 0.5 | |
| trainer: | |
| batch_size: 32 | |
| lr: 0.001 | |
| num_warmup_steps: 60000 | |
| max_training_steps: 200000 | |
| logging_steps: 2000 | |
| save_steps: 20000 | |
| eval_steps: 20000 | |