Hugging Face's logo

Spaces:
textgeflecht
/
inference-benchmarking-results-phi4-200-tokens
Running

App Files Community
textgeflecht commited on
Commit
65ceeb0
·
verified ·
1 Parent(s): 1669c52

Update results/microsoft_phi-4_2025-05-27-04-30-21.json

Browse files
Files changed (1) hide show
  1. results/microsoft_phi-4_2025-05-27-04-30-21.json +295 -295
results/microsoft_phi-4_2025-05-27-04-30-21.json CHANGED
@@ -1,296 +1,296 @@
1
- {
2
- "config": {
3
- "max_vus": 800,
4
- "duration_secs": 120,
5
- "benchmark_kind": "Rate",
6
- "warmup_duration_secs": 30,
7
- "rates": [
8
- 1.0,
9
- 10.0,
10
- 30.0,
11
- 100.0
12
- ],
13
- "num_rates": 10,
14
- "prompt_options": {
15
- "num_tokens": 200,
16
- "min_tokens": 180,
17
- "max_tokens": 220,
18
- "variance": 10
19
- },
20
- "decode_options": {
21
- "num_tokens": 200,
22
- "min_tokens": 180,
23
- "max_tokens": 220,
24
- "variance": 10
25
- },
26
- "tokenizer": "microsoft/phi-4",
27
- "model_name": "phi-4",
28
- "profile": null,
29
- "meta": null,
30
- "run_id": "llama.cpp -np 64: lmstudio-community/phi-4-GGUF:Q8_0 (200 tokens)"
31
- },
32
- "results": [
33
- {
34
- "id": "warmup",
35
- "executor_type": "ConstantVUs",
36
- "config": {
37
- "max_vus": 1,
38
- "duration_secs": 30,
39
- "rate": null
40
- },
41
- "total_requests": 13,
42
- "total_tokens": 2483,
43
- "token_throughput_secs": 79.03569838636213,
44
- "duration_ms": 31416,
45
- "time_to_first_token_ms": {
46
- "p50": 49.613,
47
- "p60": 50.523,
48
- "p70": 51.115,
49
- "p80": 51.236,
50
- "p90": 51.808,
51
- "p95": 123.175,
52
- "p99": 208.66,
53
- "avg": 63.367
54
- },
55
- "inter_token_latency_ms": {
56
- "p50": 12.405,
57
- "p60": 12.429,
58
- "p70": 12.46,
59
- "p80": 12.472,
60
- "p90": 12.492,
61
- "p95": 12.545,
62
- "p99": 12.604,
63
- "avg": 12.374
64
- },
65
- "failed_requests": 0,
66
- "successful_requests": 13,
67
- "request_rate": 0.41379946799142475,
68
- "total_tokens_sent": 2600,
69
- "e2e_latency_ms": {
70
- "p50": 2571.836,
71
- "p60": 2597.141,
72
- "p70": 2614.064,
73
- "p80": 2624.08,
74
- "p90": 2676.904,
75
- "p95": 2753.832,
76
- "p99": 2831.688,
77
- "avg": 2416.415
78
- }
79
- },
80
- {
81
- "id": "[email protected]/s",
82
- "executor_type": "ConstantArrivalRate",
83
- "config": {
84
- "max_vus": 800,
85
- "duration_secs": 120,
86
- "rate": 1.0
87
- },
88
- "total_requests": 113,
89
- "total_tokens": 21593,
90
- "token_throughput_secs": 181.58257324962796,
91
- "duration_ms": 118915,
92
- "time_to_first_token_ms": {
93
- "p50": 110.267,
94
- "p60": 113.916,
95
- "p70": 118.21,
96
- "p80": 121.21,
97
- "p90": 127.692,
98
- "p95": 133.43,
99
- "p99": 226.87,
100
- "avg": 110.162
101
- },
102
- "inter_token_latency_ms": {
103
- "p50": 28.992,
104
- "p60": 29.967,
105
- "p70": 31.02,
106
- "p80": 32.195,
107
- "p90": 32.993,
108
- "p95": 33.179,
109
- "p99": 34.066,
110
- "avg": 28.112
111
- },
112
- "failed_requests": 1,
113
- "successful_requests": 112,
114
- "request_rate": 0.941844496084765,
115
- "total_tokens_sent": 22400,
116
- "e2e_latency_ms": {
117
- "p50": 5660.775,
118
- "p60": 5938.073,
119
- "p70": 6254.077,
120
- "p80": 6437.038,
121
- "p90": 6613.518,
122
- "p95": 6703.418,
123
- "p99": 7085.261,
124
- "avg": 5527.641
125
- }
126
- },
127
- {
128
- "id": "[email protected]/s",
129
- "executor_type": "ConstantArrivalRate",
130
- "config": {
131
- "max_vus": 800,
132
- "duration_secs": 120,
133
- "rate": 10.0
134
- },
135
- "total_requests": 217,
136
- "total_tokens": 40641,
137
- "token_throughput_secs": 341.6234600560776,
138
- "duration_ms": 118964,
139
- "time_to_first_token_ms": {
140
- "p50": 20290.104,
141
- "p60": 28844.02,
142
- "p70": 42085.075,
143
- "p80": 45312.818,
144
- "p90": 52199.942,
145
- "p95": 64243.773,
146
- "p99": 68200.865,
147
- "avg": 25697.843
148
- },
149
- "inter_token_latency_ms": {
150
- "p50": 156.457,
151
- "p60": 161.461,
152
- "p70": 164.331,
153
- "p80": 168.954,
154
- "p90": 173.942,
155
- "p95": 178.544,
156
- "p99": 277.739,
157
- "avg": 151.592
158
- },
159
- "failed_requests": 1,
160
- "successful_requests": 216,
161
- "request_rate": 1.8156705635223729,
162
- "total_tokens_sent": 43200,
163
- "e2e_latency_ms": {
164
- "p50": 51939.105,
165
- "p60": 56468.128,
166
- "p70": 73270.698,
167
- "p80": 77490.554,
168
- "p90": 81749.139,
169
- "p95": 89557.185,
170
- "p99": 97354.67,
171
- "avg": 53406.626
172
- }
173
- },
174
- {
175
- "id": "[email protected]/s",
176
- "executor_type": "ConstantArrivalRate",
177
- "config": {
178
- "max_vus": 800,
179
- "duration_secs": 120,
180
- "rate": 30.0
181
- },
182
- "total_requests": 213,
183
- "total_tokens": 38720,
184
- "token_throughput_secs": 325.77208453699836,
185
- "duration_ms": 118856,
186
- "time_to_first_token_ms": {
187
- "p50": 30288.175,
188
- "p60": 34417.92,
189
- "p70": 55492.52,
190
- "p80": 60829.221,
191
- "p90": 65342.42,
192
- "p95": 74226.871,
193
- "p99": 94366.778,
194
- "avg": 34289.892
195
- },
196
- "inter_token_latency_ms": {
197
- "p50": 160.434,
198
- "p60": 165.586,
199
- "p70": 169.939,
200
- "p80": 174.12,
201
- "p90": 179.585,
202
- "p95": 183.589,
203
- "p99": 355.017,
204
- "avg": 159.812
205
- },
206
- "failed_requests": 3,
207
- "successful_requests": 210,
208
- "request_rate": 1.7668424006397123,
209
- "total_tokens_sent": 42000,
210
- "e2e_latency_ms": {
211
- "p50": 63075.813,
212
- "p60": 68000.588,
213
- "p70": 86470.699,
214
- "p80": 92136.959,
215
- "p90": 97053.809,
216
- "p95": 100845.599,
217
- "p99": 110337.984,
218
- "avg": 62993.13
219
- }
220
- },
221
- {
222
- "id": "[email protected]/s",
223
- "executor_type": "ConstantArrivalRate",
224
- "config": {
225
- "max_vus": 800,
226
- "duration_secs": 120,
227
- "rate": 100.0
228
- },
229
- "total_requests": 239,
230
- "total_tokens": 36574,
231
- "token_throughput_secs": 309.04813213647793,
232
- "duration_ms": 118344,
233
- "time_to_first_token_ms": {
234
- "p50": 37396.23,
235
- "p60": 42446.394,
236
- "p70": 52976.537,
237
- "p80": 69727.163,
238
- "p90": 75363.92,
239
- "p95": 78507.077,
240
- "p99": 87348.994,
241
- "avg": 37586.268
242
- },
243
- "inter_token_latency_ms": {
244
- "p50": 176.579,
245
- "p60": 178.323,
246
- "p70": 180.071,
247
- "p80": 186.663,
248
- "p90": 289.366,
249
- "p95": 421.137,
250
- "p99": 2381.38,
251
- "avg": 262.289
252
- },
253
- "failed_requests": 8,
254
- "successful_requests": 231,
255
- "request_rate": 1.951936307856029,
256
- "total_tokens_sent": 46200,
257
- "e2e_latency_ms": {
258
- "p50": 67206.508,
259
- "p60": 73917.829,
260
- "p70": 81426.74,
261
- "p80": 99580.194,
262
- "p90": 108583.779,
263
- "p95": 111410.062,
264
- "p99": 115243.169,
265
- "avg": 64855.285
266
- }
267
- }
268
- ],
269
- "start_time": "2025-05-27T04:16:35.210938281+00:00",
270
- "end_time": "2025-05-27T04:30:21.574896676+00:00",
271
- "system": {
272
- "cpu": [
273
- "AMD Ryzen 7 9800X3D 8-Core Processor cpu0@4699MHz",
274
- "AMD Ryzen 7 9800X3D 8-Core Processor cpu1@4699MHz",
275
- "AMD Ryzen 7 9800X3D 8-Core Processor cpu2@4699MHz",
276
- "AMD Ryzen 7 9800X3D 8-Core Processor cpu3@4699MHz",
277
- "AMD Ryzen 7 9800X3D 8-Core Processor cpu4@4699MHz",
278
- "AMD Ryzen 7 9800X3D 8-Core Processor cpu5@4699MHz",
279
- "AMD Ryzen 7 9800X3D 8-Core Processor cpu6@4699MHz",
280
- "AMD Ryzen 7 9800X3D 8-Core Processor cpu7@4699MHz",
281
- "AMD Ryzen 7 9800X3D 8-Core Processor cpu8@4699MHz",
282
- "AMD Ryzen 7 9800X3D 8-Core Processor cpu9@4699MHz",
283
- "AMD Ryzen 7 9800X3D 8-Core Processor cpu10@4699MHz",
284
- "AMD Ryzen 7 9800X3D 8-Core Processor cpu11@4699MHz",
285
- "AMD Ryzen 7 9800X3D 8-Core Processor cpu12@4699MHz",
286
- "AMD Ryzen 7 9800X3D 8-Core Processor cpu13@4699MHz",
287
- "AMD Ryzen 7 9800X3D 8-Core Processor cpu14@4699MHz",
288
- "AMD Ryzen 7 9800X3D 8-Core Processor cpu15@4699MHz"
289
- ],
290
- "memory": "83.47 GB",
291
- "os_name": "Debian GNU/Linux",
292
- "os_version": "11",
293
- "kernel": "5.15.167.4-microsoft-standard-WSL2",
294
- "hostname": "computer"
295
- }
296
  }
 
1
+ {
2
+ "config": {
3
+ "max_vus": 800,
4
+ "duration_secs": 120,
5
+ "benchmark_kind": "Rate",
6
+ "warmup_duration_secs": 30,
7
+ "rates": [
8
+ 1.0,
9
+ 10.0,
10
+ 30.0,
11
+ 100.0
12
+ ],
13
+ "num_rates": 10,
14
+ "prompt_options": {
15
+ "num_tokens": 200,
16
+ "min_tokens": 180,
17
+ "max_tokens": 220,
18
+ "variance": 10
19
+ },
20
+ "decode_options": {
21
+ "num_tokens": 200,
22
+ "min_tokens": 180,
23
+ "max_tokens": 220,
24
+ "variance": 10
25
+ },
26
+ "tokenizer": "microsoft/phi-4",
27
+ "model_name": "phi-4",
28
+ "profile": null,
29
+ "meta": null,
30
+ "run_id": "llama.cpp -np 64: unsloth/phi-4-GGUF:Q8_0 (200 tokens)"
31
+ },
32
+ "results": [
33
+ {
34
+ "id": "warmup",
35
+ "executor_type": "ConstantVUs",
36
+ "config": {
37
+ "max_vus": 1,
38
+ "duration_secs": 30,
39
+ "rate": null
40
+ },
41
+ "total_requests": 13,
42
+ "total_tokens": 2483,
43
+ "token_throughput_secs": 79.03569838636213,
44
+ "duration_ms": 31416,
45
+ "time_to_first_token_ms": {
46
+ "p50": 49.613,
47
+ "p60": 50.523,
48
+ "p70": 51.115,
49
+ "p80": 51.236,
50
+ "p90": 51.808,
51
+ "p95": 123.175,
52
+ "p99": 208.66,
53
+ "avg": 63.367
54
+ },
55
+ "inter_token_latency_ms": {
56
+ "p50": 12.405,
57
+ "p60": 12.429,
58
+ "p70": 12.46,
59
+ "p80": 12.472,
60
+ "p90": 12.492,
61
+ "p95": 12.545,
62
+ "p99": 12.604,
63
+ "avg": 12.374
64
+ },
65
+ "failed_requests": 0,
66
+ "successful_requests": 13,
67
+ "request_rate": 0.41379946799142475,
68
+ "total_tokens_sent": 2600,
69
+ "e2e_latency_ms": {
70
+ "p50": 2571.836,
71
+ "p60": 2597.141,
72
+ "p70": 2614.064,
73
+ "p80": 2624.08,
74
+ "p90": 2676.904,
75
+ "p95": 2753.832,
76
+ "p99": 2831.688,
77
+ "avg": 2416.415
78
+ }
79
+ },
80
+ {
81
+ "id": "[email protected]/s",
82
+ "executor_type": "ConstantArrivalRate",
83
+ "config": {
84
+ "max_vus": 800,
85
+ "duration_secs": 120,
86
+ "rate": 1.0
87
+ },
88
+ "total_requests": 113,
89
+ "total_tokens": 21593,
90
+ "token_throughput_secs": 181.58257324962796,
91
+ "duration_ms": 118915,
92
+ "time_to_first_token_ms": {
93
+ "p50": 110.267,
94
+ "p60": 113.916,
95
+ "p70": 118.21,
96
+ "p80": 121.21,
97
+ "p90": 127.692,
98
+ "p95": 133.43,
99
+ "p99": 226.87,
100
+ "avg": 110.162
101
+ },
102
+ "inter_token_latency_ms": {
103
+ "p50": 28.992,
104
+ "p60": 29.967,
105
+ "p70": 31.02,
106
+ "p80": 32.195,
107
+ "p90": 32.993,
108
+ "p95": 33.179,
109
+ "p99": 34.066,
110
+ "avg": 28.112
111
+ },
112
+ "failed_requests": 1,
113
+ "successful_requests": 112,
114
+ "request_rate": 0.941844496084765,
115
+ "total_tokens_sent": 22400,
116
+ "e2e_latency_ms": {
117
+ "p50": 5660.775,
118
+ "p60": 5938.073,
119
+ "p70": 6254.077,
120
+ "p80": 6437.038,
121
+ "p90": 6613.518,
122
+ "p95": 6703.418,
123
+ "p99": 7085.261,
124
+ "avg": 5527.641
125
+ }
126
+ },
127
+ {
128
+ "id": "[email protected]/s",
129
+ "executor_type": "ConstantArrivalRate",
130
+ "config": {
131
+ "max_vus": 800,
132
+ "duration_secs": 120,
133
+ "rate": 10.0
134
+ },
135
+ "total_requests": 217,
136
+ "total_tokens": 40641,
137
+ "token_throughput_secs": 341.6234600560776,
138
+ "duration_ms": 118964,
139
+ "time_to_first_token_ms": {
140
+ "p50": 20290.104,
141
+ "p60": 28844.02,
142
+ "p70": 42085.075,
143
+ "p80": 45312.818,
144
+ "p90": 52199.942,
145
+ "p95": 64243.773,
146
+ "p99": 68200.865,
147
+ "avg": 25697.843
148
+ },
149
+ "inter_token_latency_ms": {
150
+ "p50": 156.457,
151
+ "p60": 161.461,
152
+ "p70": 164.331,
153
+ "p80": 168.954,
154
+ "p90": 173.942,
155
+ "p95": 178.544,
156
+ "p99": 277.739,
157
+ "avg": 151.592
158
+ },
159
+ "failed_requests": 1,
160
+ "successful_requests": 216,
161
+ "request_rate": 1.8156705635223729,
162
+ "total_tokens_sent": 43200,
163
+ "e2e_latency_ms": {
164
+ "p50": 51939.105,
165
+ "p60": 56468.128,
166
+ "p70": 73270.698,
167
+ "p80": 77490.554,
168
+ "p90": 81749.139,
169
+ "p95": 89557.185,
170
+ "p99": 97354.67,
171
+ "avg": 53406.626
172
+ }
173
+ },
174
+ {
175
+ "id": "[email protected]/s",
176
+ "executor_type": "ConstantArrivalRate",
177
+ "config": {
178
+ "max_vus": 800,
179
+ "duration_secs": 120,
180
+ "rate": 30.0
181
+ },
182
+ "total_requests": 213,
183
+ "total_tokens": 38720,
184
+ "token_throughput_secs": 325.77208453699836,
185
+ "duration_ms": 118856,
186
+ "time_to_first_token_ms": {
187
+ "p50": 30288.175,
188
+ "p60": 34417.92,
189
+ "p70": 55492.52,
190
+ "p80": 60829.221,
191
+ "p90": 65342.42,
192
+ "p95": 74226.871,
193
+ "p99": 94366.778,
194
+ "avg": 34289.892
195
+ },
196
+ "inter_token_latency_ms": {
197
+ "p50": 160.434,
198
+ "p60": 165.586,
199
+ "p70": 169.939,
200
+ "p80": 174.12,
201
+ "p90": 179.585,
202
+ "p95": 183.589,
203
+ "p99": 355.017,
204
+ "avg": 159.812
205
+ },
206
+ "failed_requests": 3,
207
+ "successful_requests": 210,
208
+ "request_rate": 1.7668424006397123,
209
+ "total_tokens_sent": 42000,
210
+ "e2e_latency_ms": {
211
+ "p50": 63075.813,
212
+ "p60": 68000.588,
213
+ "p70": 86470.699,
214
+ "p80": 92136.959,
215
+ "p90": 97053.809,
216
+ "p95": 100845.599,
217
+ "p99": 110337.984,
218
+ "avg": 62993.13
219
+ }
220
+ },
221
+ {
222
+ "id": "[email protected]/s",
223
+ "executor_type": "ConstantArrivalRate",
224
+ "config": {
225
+ "max_vus": 800,
226
+ "duration_secs": 120,
227
+ "rate": 100.0
228
+ },
229
+ "total_requests": 239,
230
+ "total_tokens": 36574,
231
+ "token_throughput_secs": 309.04813213647793,
232
+ "duration_ms": 118344,
233
+ "time_to_first_token_ms": {
234
+ "p50": 37396.23,
235
+ "p60": 42446.394,
236
+ "p70": 52976.537,
237
+ "p80": 69727.163,
238
+ "p90": 75363.92,
239
+ "p95": 78507.077,
240
+ "p99": 87348.994,
241
+ "avg": 37586.268
242
+ },
243
+ "inter_token_latency_ms": {
244
+ "p50": 176.579,
245
+ "p60": 178.323,
246
+ "p70": 180.071,
247
+ "p80": 186.663,
248
+ "p90": 289.366,
249
+ "p95": 421.137,
250
+ "p99": 2381.38,
251
+ "avg": 262.289
252
+ },
253
+ "failed_requests": 8,
254
+ "successful_requests": 231,
255
+ "request_rate": 1.951936307856029,
256
+ "total_tokens_sent": 46200,
257
+ "e2e_latency_ms": {
258
+ "p50": 67206.508,
259
+ "p60": 73917.829,
260
+ "p70": 81426.74,
261
+ "p80": 99580.194,
262
+ "p90": 108583.779,
263
+ "p95": 111410.062,
264
+ "p99": 115243.169,
265
+ "avg": 64855.285
266
+ }
267
+ }
268
+ ],
269
+ "start_time": "2025-05-27T04:16:35.210938281+00:00",
270
+ "end_time": "2025-05-27T04:30:21.574896676+00:00",
271
+ "system": {
272
+ "cpu": [
273
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu0@4699MHz",
274
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu1@4699MHz",
275
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu2@4699MHz",
276
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu3@4699MHz",
277
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu4@4699MHz",
278
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu5@4699MHz",
279
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu6@4699MHz",
280
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu7@4699MHz",
281
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu8@4699MHz",
282
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu9@4699MHz",
283
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu10@4699MHz",
284
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu11@4699MHz",
285
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu12@4699MHz",
286
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu13@4699MHz",
287
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu14@4699MHz",
288
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu15@4699MHz"
289
+ ],
290
+ "memory": "83.47 GB",
291
+ "os_name": "Debian GNU/Linux",
292
+ "os_version": "11",
293
+ "kernel": "5.15.167.4-microsoft-standard-WSL2",
294
+ "hostname": "computer"
295
+ }
296
  }