Hugging Face's logo

PowerServe
/
Llama-3.1-8B-PowerServe-QNN29-8G4

GGUF
Inference Endpoints
conversational
Model card Files Community
xin-tong commited on
Commit
d0c9884
·
verified ·
1 Parent(s): a90b81e

Upload PowerServe/Llama-3.1-8B-PowerServe-QNN29-8G4

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. ggml/weights.gguf +2 -2
  2. model.json +16 -16
  3. qnn/config.json +85 -29
  4. qnn/kv/layer_0_key_0.raw +0 -0
  5. qnn/kv/layer_0_key_1.raw +0 -0
  6. qnn/kv/layer_0_key_2.raw +0 -0
  7. qnn/kv/layer_0_key_3.raw +0 -0
  8. qnn/kv/layer_0_key_4.raw +0 -0
  9. qnn/kv/layer_0_key_5.raw +0 -0
  10. qnn/kv/layer_0_key_6.raw +0 -0
  11. qnn/kv/layer_0_key_7.raw +0 -0
  12. qnn/kv/layer_0_value_0.raw +0 -0
  13. qnn/kv/layer_0_value_1.raw +0 -0
  14. qnn/kv/layer_0_value_2.raw +0 -0
  15. qnn/kv/layer_0_value_3.raw +0 -0
  16. qnn/kv/layer_0_value_4.raw +0 -0
  17. qnn/kv/layer_0_value_5.raw +0 -0
  18. qnn/kv/layer_0_value_6.raw +0 -0
  19. qnn/kv/layer_0_value_7.raw +0 -0
  20. qnn/kv/layer_10_key_0.raw +0 -0
  21. qnn/kv/layer_10_key_1.raw +0 -0
  22. qnn/kv/layer_10_key_2.raw +0 -0
  23. qnn/kv/layer_10_key_3.raw +0 -0
  24. qnn/kv/layer_10_key_4.raw +0 -0
  25. qnn/kv/layer_10_key_5.raw +0 -0
  26. qnn/kv/layer_10_key_6.raw +0 -0
  27. qnn/kv/layer_10_key_7.raw +0 -0
  28. qnn/kv/layer_10_value_0.raw +0 -0
  29. qnn/kv/layer_10_value_1.raw +0 -0
  30. qnn/kv/layer_10_value_2.raw +0 -0
  31. qnn/kv/layer_10_value_3.raw +0 -0
  32. qnn/kv/layer_10_value_4.raw +0 -0
  33. qnn/kv/layer_10_value_5.raw +0 -0
  34. qnn/kv/layer_10_value_6.raw +0 -0
  35. qnn/kv/layer_10_value_7.raw +0 -0
  36. qnn/kv/layer_11_key_0.raw +0 -0
  37. qnn/kv/layer_11_key_1.raw +0 -0
  38. qnn/kv/layer_11_key_2.raw +0 -0
  39. qnn/kv/layer_11_key_3.raw +0 -0
  40. qnn/kv/layer_11_key_4.raw +0 -0
  41. qnn/kv/layer_11_key_5.raw +0 -0
  42. qnn/kv/layer_11_key_6.raw +0 -0
  43. qnn/kv/layer_11_key_7.raw +0 -0
  44. qnn/kv/layer_11_value_0.raw +0 -0
  45. qnn/kv/layer_11_value_1.raw +0 -0
  46. qnn/kv/layer_11_value_2.raw +0 -0
  47. qnn/kv/layer_11_value_3.raw +0 -0
  48. qnn/kv/layer_11_value_4.raw +0 -0
  49. qnn/kv/layer_11_value_5.raw +0 -0
  50. qnn/kv/layer_11_value_6.raw +0 -0
ggml/weights.gguf CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:311af5e718008431b60b8dc4c3613e835f04d4168ddd97c167277bf795657927
3
- size 1917656096
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9b2a3f5861f15178ac12104c8b0206c40f7591358e6cf5bc0f423965df718ac8
3
+ size 4525777760
model.json CHANGED
@@ -1,26 +1,26 @@
1
  {
2
- "model_id": "smallthinker_0.5b_8gen4",
3
- "model_arch": "qwen2",
4
  "version": 20241202,
5
  "llm_config": {
6
- "embed_dim": 896,
7
- "ffn_dim": 4864,
8
- "head_size": 64,
9
- "kv_dim": 128,
10
- "n_attn_heads": 14,
11
- "n_attn_kv_heads": 2,
12
- "n_ctx": 32768,
13
- "n_layers": 24,
14
- "norm_eps": 9.999999974752427e-07,
15
- "vocab_size": 151936,
16
  "rope_config": {
17
- "n_rope_ctx_orig": 32768,
18
  "rope_attn_factor": 1.0,
19
- "rope_dim": 64,
20
- "rope_freq_base": 1000000.0,
21
  "rope_freq_scale": 1.0,
22
  "rope_scale_type": "linear",
23
- "rope_type": -1
24
  }
25
  },
26
  "vision": {}
 
1
  {
2
+ "model_id": "llama3.1_8b_8gen4",
3
+ "model_arch": "llama",
4
  "version": 20241202,
5
  "llm_config": {
6
+ "embed_dim": 4096,
7
+ "ffn_dim": 14336,
8
+ "head_size": 128,
9
+ "kv_dim": 1024,
10
+ "n_attn_heads": 32,
11
+ "n_attn_kv_heads": 8,
12
+ "n_ctx": 131072,
13
+ "n_layers": 32,
14
+ "norm_eps": 9.999999747378752e-06,
15
+ "vocab_size": 128256,
16
  "rope_config": {
17
+ "n_rope_ctx_orig": 131072,
18
  "rope_attn_factor": 1.0,
19
+ "rope_dim": 128,
20
+ "rope_freq_base": 500000.0,
21
  "rope_freq_scale": 1.0,
22
  "rope_scale_type": "linear",
23
+ "rope_type": 0
24
  }
25
  },
26
  "vision": {}
qnn/config.json CHANGED
@@ -1,15 +1,15 @@
1
  {
2
  "model_parameters": {
3
- "n_layers": 36,
4
- "vocab_size": 151936,
5
- "embed_dim": 2048,
6
- "ffn_hidden_dim": 11008,
7
  "head_dim": 128,
8
- "n_kv_heads": 2,
9
- "rope_theta": 1000000.0,
10
- "rms_norm_eps": 1e-06,
11
- "attention_mask_value": -50000.0,
12
- "tie_embedding": true
13
  },
14
  "qnn_parameters": {
15
  "n_hvx_threads": 4
@@ -18,65 +18,121 @@
18
  {
19
  "type": "transformers",
20
  "start_layer_id": 0,
21
- "end_layer_id": 18,
22
- "batch_size": 16,
23
  "cache_size": 1920,
24
  "context_size": 2048,
25
- "graph_name": "batch_16",
26
- "model_path": "smallthinker_3b_0.bin",
27
  "kv_path_format": "kv/layer_{layer_id}_{kv_type}_{head_id}.raw",
28
- "kv_size": 11,
29
  "x_name": "x",
30
  "out_name": "out"
31
  },
32
  {
33
  "type": "transformers",
34
- "start_layer_id": 18,
35
- "end_layer_id": 36,
36
- "batch_size": 16,
37
  "cache_size": 1920,
38
  "context_size": 2048,
39
- "graph_name": "batch_16",
40
- "model_path": "smallthinker_3b_1.bin",
41
  "kv_path_format": "kv/layer_{layer_id}_{kv_type}_{head_id}.raw",
42
- "kv_size": 11,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
43
  "x_name": "x",
44
  "out_name": "out"
45
  },
46
  {
47
  "type": "transformers",
48
  "start_layer_id": 0,
49
- "end_layer_id": 18,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
50
  "batch_size": 128,
51
  "cache_size": 1920,
52
  "context_size": 2048,
53
  "graph_name": "batch_128",
54
- "model_path": "smallthinker_3b_0.bin",
55
  "kv_path_format": "kv/layer_{layer_id}_{kv_type}_{head_id}.raw",
56
- "kv_size": 11,
57
  "x_name": "x",
58
  "out_name": "out"
59
  },
60
  {
61
  "type": "transformers",
62
- "start_layer_id": 18,
63
- "end_layer_id": 36,
64
  "batch_size": 128,
65
  "cache_size": 1920,
66
  "context_size": 2048,
67
  "graph_name": "batch_128",
68
- "model_path": "smallthinker_3b_1.bin",
69
  "kv_path_format": "kv/layer_{layer_id}_{kv_type}_{head_id}.raw",
70
- "kv_size": 11,
71
  "x_name": "x",
72
  "out_name": "out"
73
  }
74
  ],
75
  "embeddings": [
76
  {
77
- "graph_name": "batch_16",
78
  "model_path": "lm_head.bin",
79
- "batch_size": 16,
80
  "x_name": "x",
81
  "out_name": "logits"
82
  },
 
1
  {
2
  "model_parameters": {
3
+ "n_layers": 32,
4
+ "vocab_size": 128256,
5
+ "embed_dim": 4096,
6
+ "ffn_hidden_dim": 14336,
7
  "head_dim": 128,
8
+ "n_kv_heads": 8,
9
+ "rope_theta": 500000.0,
10
+ "rms_norm_eps": 1e-05,
11
+ "attention_mask_value": -100000.0,
12
+ "tie_embedding": false
13
  },
14
  "qnn_parameters": {
15
  "n_hvx_threads": 4
 
18
  {
19
  "type": "transformers",
20
  "start_layer_id": 0,
21
+ "end_layer_id": 8,
22
+ "batch_size": 1,
23
  "cache_size": 1920,
24
  "context_size": 2048,
25
+ "graph_name": "batch_1",
26
+ "model_path": "llama3_1_8b_0.bin",
27
  "kv_path_format": "kv/layer_{layer_id}_{kv_type}_{head_id}.raw",
28
+ "kv_size": 13,
29
  "x_name": "x",
30
  "out_name": "out"
31
  },
32
  {
33
  "type": "transformers",
34
+ "start_layer_id": 8,
35
+ "end_layer_id": 16,
36
+ "batch_size": 1,
37
  "cache_size": 1920,
38
  "context_size": 2048,
39
+ "graph_name": "batch_1",
40
+ "model_path": "llama3_1_8b_1.bin",
41
  "kv_path_format": "kv/layer_{layer_id}_{kv_type}_{head_id}.raw",
42
+ "kv_size": 13,
43
+ "x_name": "x",
44
+ "out_name": "out"
45
+ },
46
+ {
47
+ "type": "transformers",
48
+ "start_layer_id": 16,
49
+ "end_layer_id": 24,
50
+ "batch_size": 1,
51
+ "cache_size": 1920,
52
+ "context_size": 2048,
53
+ "graph_name": "batch_1",
54
+ "model_path": "llama3_1_8b_2.bin",
55
+ "kv_path_format": "kv/layer_{layer_id}_{kv_type}_{head_id}.raw",
56
+ "kv_size": 13,
57
+ "x_name": "x",
58
+ "out_name": "out"
59
+ },
60
+ {
61
+ "type": "transformers",
62
+ "start_layer_id": 24,
63
+ "end_layer_id": 32,
64
+ "batch_size": 1,
65
+ "cache_size": 1920,
66
+ "context_size": 2048,
67
+ "graph_name": "batch_1",
68
+ "model_path": "llama3_1_8b_3.bin",
69
+ "kv_path_format": "kv/layer_{layer_id}_{kv_type}_{head_id}.raw",
70
+ "kv_size": 13,
71
  "x_name": "x",
72
  "out_name": "out"
73
  },
74
  {
75
  "type": "transformers",
76
  "start_layer_id": 0,
77
+ "end_layer_id": 8,
78
+ "batch_size": 128,
79
+ "cache_size": 1920,
80
+ "context_size": 2048,
81
+ "graph_name": "batch_128",
82
+ "model_path": "llama3_1_8b_0.bin",
83
+ "kv_path_format": "kv/layer_{layer_id}_{kv_type}_{head_id}.raw",
84
+ "kv_size": 13,
85
+ "x_name": "x",
86
+ "out_name": "out"
87
+ },
88
+ {
89
+ "type": "transformers",
90
+ "start_layer_id": 8,
91
+ "end_layer_id": 16,
92
+ "batch_size": 128,
93
+ "cache_size": 1920,
94
+ "context_size": 2048,
95
+ "graph_name": "batch_128",
96
+ "model_path": "llama3_1_8b_1.bin",
97
+ "kv_path_format": "kv/layer_{layer_id}_{kv_type}_{head_id}.raw",
98
+ "kv_size": 13,
99
+ "x_name": "x",
100
+ "out_name": "out"
101
+ },
102
+ {
103
+ "type": "transformers",
104
+ "start_layer_id": 16,
105
+ "end_layer_id": 24,
106
  "batch_size": 128,
107
  "cache_size": 1920,
108
  "context_size": 2048,
109
  "graph_name": "batch_128",
110
+ "model_path": "llama3_1_8b_2.bin",
111
  "kv_path_format": "kv/layer_{layer_id}_{kv_type}_{head_id}.raw",
112
+ "kv_size": 13,
113
  "x_name": "x",
114
  "out_name": "out"
115
  },
116
  {
117
  "type": "transformers",
118
+ "start_layer_id": 24,
119
+ "end_layer_id": 32,
120
  "batch_size": 128,
121
  "cache_size": 1920,
122
  "context_size": 2048,
123
  "graph_name": "batch_128",
124
+ "model_path": "llama3_1_8b_3.bin",
125
  "kv_path_format": "kv/layer_{layer_id}_{kv_type}_{head_id}.raw",
126
+ "kv_size": 13,
127
  "x_name": "x",
128
  "out_name": "out"
129
  }
130
  ],
131
  "embeddings": [
132
  {
133
+ "graph_name": "batch_1",
134
  "model_path": "lm_head.bin",
135
+ "batch_size": 1,
136
  "x_name": "x",
137
  "out_name": "logits"
138
  },
qnn/kv/layer_0_key_0.raw CHANGED
Binary files a/qnn/kv/layer_0_key_0.raw and b/qnn/kv/layer_0_key_0.raw differ
 
qnn/kv/layer_0_key_1.raw CHANGED
Binary files a/qnn/kv/layer_0_key_1.raw and b/qnn/kv/layer_0_key_1.raw differ
 
qnn/kv/layer_0_key_2.raw ADDED
Binary file (6.66 kB). View file
 
qnn/kv/layer_0_key_3.raw ADDED
Binary file (6.66 kB). View file
 
qnn/kv/layer_0_key_4.raw ADDED
Binary file (6.66 kB). View file
 
qnn/kv/layer_0_key_5.raw ADDED
Binary file (6.66 kB). View file
 
qnn/kv/layer_0_key_6.raw ADDED
Binary file (6.66 kB). View file
 
qnn/kv/layer_0_key_7.raw ADDED
Binary file (6.66 kB). View file
 
qnn/kv/layer_0_value_0.raw CHANGED
Binary files a/qnn/kv/layer_0_value_0.raw and b/qnn/kv/layer_0_value_0.raw differ
 
qnn/kv/layer_0_value_1.raw CHANGED
Binary files a/qnn/kv/layer_0_value_1.raw and b/qnn/kv/layer_0_value_1.raw differ
 
qnn/kv/layer_0_value_2.raw ADDED
Binary file (6.66 kB). View file
 
qnn/kv/layer_0_value_3.raw ADDED
Binary file (6.66 kB). View file
 
qnn/kv/layer_0_value_4.raw ADDED
Binary file (6.66 kB). View file
 
qnn/kv/layer_0_value_5.raw ADDED
Binary file (6.66 kB). View file
 
qnn/kv/layer_0_value_6.raw ADDED
Binary file (6.66 kB). View file
 
qnn/kv/layer_0_value_7.raw ADDED
Binary file (6.66 kB). View file
 
qnn/kv/layer_10_key_0.raw CHANGED
Binary files a/qnn/kv/layer_10_key_0.raw and b/qnn/kv/layer_10_key_0.raw differ
 
qnn/kv/layer_10_key_1.raw CHANGED
Binary files a/qnn/kv/layer_10_key_1.raw and b/qnn/kv/layer_10_key_1.raw differ
 
qnn/kv/layer_10_key_2.raw ADDED
Binary file (6.66 kB). View file
 
qnn/kv/layer_10_key_3.raw ADDED
Binary file (6.66 kB). View file
 
qnn/kv/layer_10_key_4.raw ADDED
Binary file (6.66 kB). View file
 
qnn/kv/layer_10_key_5.raw ADDED
Binary file (6.66 kB). View file
 
qnn/kv/layer_10_key_6.raw ADDED
Binary file (6.66 kB). View file
 
qnn/kv/layer_10_key_7.raw ADDED
Binary file (6.66 kB). View file
 
qnn/kv/layer_10_value_0.raw CHANGED
Binary files a/qnn/kv/layer_10_value_0.raw and b/qnn/kv/layer_10_value_0.raw differ
 
qnn/kv/layer_10_value_1.raw CHANGED
Binary files a/qnn/kv/layer_10_value_1.raw and b/qnn/kv/layer_10_value_1.raw differ
 
qnn/kv/layer_10_value_2.raw ADDED
Binary file (6.66 kB). View file
 
qnn/kv/layer_10_value_3.raw ADDED
Binary file (6.66 kB). View file
 
qnn/kv/layer_10_value_4.raw ADDED
Binary file (6.66 kB). View file
 
qnn/kv/layer_10_value_5.raw ADDED
Binary file (6.66 kB). View file
 
qnn/kv/layer_10_value_6.raw ADDED
Binary file (6.66 kB). View file
 
qnn/kv/layer_10_value_7.raw ADDED
Binary file (6.66 kB). View file
 
qnn/kv/layer_11_key_0.raw CHANGED
Binary files a/qnn/kv/layer_11_key_0.raw and b/qnn/kv/layer_11_key_0.raw differ
 
qnn/kv/layer_11_key_1.raw CHANGED
Binary files a/qnn/kv/layer_11_key_1.raw and b/qnn/kv/layer_11_key_1.raw differ
 
qnn/kv/layer_11_key_2.raw ADDED
Binary file (6.66 kB). View file
 
qnn/kv/layer_11_key_3.raw ADDED
Binary file (6.66 kB). View file
 
qnn/kv/layer_11_key_4.raw ADDED
Binary file (6.66 kB). View file
 
qnn/kv/layer_11_key_5.raw ADDED
Binary file (6.66 kB). View file
 
qnn/kv/layer_11_key_6.raw ADDED
Binary file (6.66 kB). View file
 
qnn/kv/layer_11_key_7.raw ADDED
Binary file (6.66 kB). View file
 
qnn/kv/layer_11_value_0.raw CHANGED
Binary files a/qnn/kv/layer_11_value_0.raw and b/qnn/kv/layer_11_value_0.raw differ
 
qnn/kv/layer_11_value_1.raw CHANGED
Binary files a/qnn/kv/layer_11_value_1.raw and b/qnn/kv/layer_11_value_1.raw differ
 
qnn/kv/layer_11_value_2.raw ADDED
Binary file (6.66 kB). View file
 
qnn/kv/layer_11_value_3.raw ADDED
Binary file (6.66 kB). View file
 
qnn/kv/layer_11_value_4.raw ADDED
Binary file (6.66 kB). View file
 
qnn/kv/layer_11_value_5.raw ADDED
Binary file (6.66 kB). View file
 
qnn/kv/layer_11_value_6.raw ADDED
Binary file (6.66 kB). View file