lewtun HF Staff commited on
Commit
00f59e2
·
verified ·
1 Parent(s): 6ef6ad6

Extend RoPE theta to 300k and extend position embeddings to 32k

Browse files
Files changed (1) hide show
  1. config.json +4 -4
config.json CHANGED
@@ -9,20 +9,20 @@
9
  "hidden_size": 3584,
10
  "initializer_range": 0.02,
11
  "intermediate_size": 18944,
12
- "max_position_embeddings": 4096,
13
  "max_window_layers": 28,
14
  "model_type": "qwen2",
15
  "num_attention_heads": 28,
16
  "num_hidden_layers": 28,
17
  "num_key_value_heads": 4,
18
  "rms_norm_eps": 1e-06,
19
- "rope_theta": 10000,
20
- "sliding_window": 4096,
 
21
  "tie_word_embeddings": false,
22
  "torch_dtype": "bfloat16",
23
  "transformers_version": "4.44.0",
24
  "use_cache": true,
25
- "use_mrope": false,
26
  "use_sliding_window": false,
27
  "vocab_size": 152064
28
  }
 
9
  "hidden_size": 3584,
10
  "initializer_range": 0.02,
11
  "intermediate_size": 18944,
12
+ "max_position_embeddings": 32768,
13
  "max_window_layers": 28,
14
  "model_type": "qwen2",
15
  "num_attention_heads": 28,
16
  "num_hidden_layers": 28,
17
  "num_key_value_heads": 4,
18
  "rms_norm_eps": 1e-06,
19
+ "rope_scaling": null,
20
+ "rope_theta": 300000.0,
21
+ "sliding_window": null,
22
  "tie_word_embeddings": false,
23
  "torch_dtype": "bfloat16",
24
  "transformers_version": "4.44.0",
25
  "use_cache": true,
 
26
  "use_sliding_window": false,
27
  "vocab_size": 152064
28
  }