File size: 4,062 Bytes
930eca4 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 |
{
"name": "blt_main_entropy_100m_512w",
"dump_dir": "/checkpoints/blt_main_entropy_100m_512w",
"seed": 42,
"debug_dynamo": false,
"grad_acc_steps": 1,
"gc_collect_freq": 1000,
"probe_freq": null,
"steps": 100000,
"max_steps": null,
"data": {
"s3_profile": null,
"batch_size": 8,
"seq_len": 8192,
"seed": 42,
"add_bos": true,
"add_eos": true,
"load_async": false,
"prefetch_size": 20,
"preprocess_dir": "/corpora/entropy_preprocess",
"dataset_files": null,
"entropy_model_name": "transformer_100m",
"arrow_batch_size": 100,
"buffer_size": 512,
"pad_to_max_length": true,
"max_encoder_seq_length": 8192,
"enable_byte_ngrams": false,
"add_patches": false,
"tokenizer_args": {
"name": "blt",
"init_kwargs": null
},
"patcher_args": {
"patching_mode": "byte",
"patching_device": "cuda",
"entropy_model_checkpoint_dir": null,
"realtime_patching": false,
"threshold": 1.335442066192627,
"threshold_add": null,
"max_patch_length": null,
"patch_size": 4.5,
"patching_batch_size": 1,
"device": "cuda",
"monotonicity": false,
"log_time": false
}
},
"optim": {
"lr": 0.0004,
"weight_decay": 0.1,
"epsilon": 1e-08,
"beta1": 0.9,
"beta2": 0.95,
"clip": 10.0,
"scheduler": "cosine",
"warmup": 500,
"lr_min_ratio": 0.1,
"cycle_length": 1.0,
"cosine_theta": 1.0,
"annealing_step": 1000,
"decay_fraction": 0.1,
"exp_factor": 0.5
},
"model": null,
"entropy_model": {
"dim": 768,
"n_layers": 14,
"head_dim": null,
"n_heads": 12,
"n_kv_heads": null,
"ffn_dim_multiplier": 1.0,
"multiple_of": 256,
"norm_eps": 1e-05,
"rope_theta": 10000.0,
"rope_use_fp32_in_outer_product": false,
"init_base_std": null,
"init_std_factor": "current_depth",
"max_seqlen": 8192,
"attn_impl": "xformers",
"attn_bias_type": "local_block_causal",
"eos_id": 2,
"seed": 42,
"vocab_size": 260,
"weight_tying": false,
"sliding_window": 512
},
"train_entropy_model": true,
"distributed": {
"dp_shard": 1,
"dp_replicate": 8,
"tp_size": 1,
"selective_activation_checkpointing": false,
"compile": false,
"fsdp_type": "full_shard",
"model_dtype": "bf16",
"float8_recipe": null,
"float8_filter": "layers\\.[0-9]+\\.",
"matmul_allow_tf32": false,
"allow_bf16_reduced_precision_reduction": true,
"detect_anomaly": false,
"compile_cache_size_limit": 8,
"spawn_method": "forkserver"
},
"env": {
"MKL_SERVICE_FORCE_INTEL": "GNU",
"OMP_NUM_THREADS": "1",
"MKL_NUM_THREADS": "1",
"ENABLE_INTRA_NODE_COMM": "1",
"TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
"NCCL_IB_TIMEOUT": "22",
"NCCL_DEBUG": "INFO",
"TORCH_NCCL_ASYNC_ERROR_HANDLING": "1"
},
"checkpoint": {
"dump": {
"every": 500,
"keep": 3
},
"eval": {
"every": 1000,
"keep": -1
},
"path": "/checkpoints/blt_main_entropy_100m_512w/checkpoints",
"init_ckpt_path": null,
"continue_training_from_init": false,
"s3_profile": null
},
"profiling": {
"run": false,
"trace_folder": "profiling",
"mem_warmup": 100,
"mem_steps": 2,
"profile_warmup": 102,
"profile_steps": 2
},
"logging": {
"freq": 10,
"acc_freq": null,
"wandb": null
},
"async_eval_gpus": null,
"eval": null,
"eval_on_gpus": 8
}
|