replace router_aux_loss_coef with moe_loss_weight
Browse files- config.json +1 -1
config.json
CHANGED
|
@@ -21,7 +21,7 @@
|
|
| 21 |
"moe_jitter_eps": 0,
|
| 22 |
"moe_num_experts": 16,
|
| 23 |
"moe_top_k": 4,
|
| 24 |
-
"
|
| 25 |
},
|
| 26 |
"initializer_range": 0.02,
|
| 27 |
"max_seq_len": 32768,
|
|
|
|
| 21 |
"moe_jitter_eps": 0,
|
| 22 |
"moe_num_experts": 16,
|
| 23 |
"moe_top_k": 4,
|
| 24 |
+
"moe_loss_weight": 0.05
|
| 25 |
},
|
| 26 |
"initializer_range": 0.02,
|
| 27 |
"max_seq_len": 32768,
|