Hugging Face's logo

Spaces:
textgeflecht
/
inference-benchmarking-results-phi4-200-tokens
Running

App Files Community
textgeflecht commited on
Commit
ecea808
·
verified ·
1 Parent(s): 2d857d6

Upload RedHatAI_phi-4-FP8-dynamic_2025-05-27-14-24-12.json

Browse files

sudo docker run --runtime=nvidia --gpus all --net=host --ipc=host \
-v ~/.cache/huggingface:/root/.cache/huggingface \
-e HUGGING_FACE_HUB_TOKEN=$HUGGING_FACE_HUB_TOKEN \
myvllm-kaln27-updated \
--model RedHatAI/phi-4-FP8-dynamic \
--max-model-len 16384 \
--enable-prefix-caching \
--enable-chunked-prefill \
--disable-log-requests

Files changed (1) hide show
  1. results/RedHatAI_phi-4-FP8-dynamic_2025-05-27-14-24-12.json +296 -0
results/RedHatAI_phi-4-FP8-dynamic_2025-05-27-14-24-12.json ADDED
@@ -0,0 +1,296 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "config": {
3
+ "max_vus": 800,
4
+ "duration_secs": 120,
5
+ "benchmark_kind": "Rate",
6
+ "warmup_duration_secs": 30,
7
+ "rates": [
8
+ 1.0,
9
+ 10.0,
10
+ 30.0,
11
+ 100.0
12
+ ],
13
+ "num_rates": 10,
14
+ "prompt_options": {
15
+ "num_tokens": 200,
16
+ "min_tokens": 180,
17
+ "max_tokens": 220,
18
+ "variance": 10
19
+ },
20
+ "decode_options": {
21
+ "num_tokens": 200,
22
+ "min_tokens": 180,
23
+ "max_tokens": 220,
24
+ "variance": 10
25
+ },
26
+ "tokenizer": "RedHatAI/phi-4-FP8-dynamic",
27
+ "model_name": "phi-4",
28
+ "profile": null,
29
+ "meta": null,
30
+ "run_id": "vLLM --enable-prefix-caching --enable-chunked-prefill: RedHatAI/phi-4-FP8-dynamic (200 tokens)"
31
+ },
32
+ "results": [
33
+ {
34
+ "id": "warmup",
35
+ "executor_type": "ConstantVUs",
36
+ "config": {
37
+ "max_vus": 1,
38
+ "duration_secs": 30,
39
+ "rate": null
40
+ },
41
+ "total_requests": 8,
42
+ "total_tokens": 1295,
43
+ "token_throughput_secs": 39.49929335141966,
44
+ "duration_ms": 32785,
45
+ "time_to_first_token_ms": {
46
+ "p50": 31.473,
47
+ "p60": 31.576,
48
+ "p70": 31.827,
49
+ "p80": 32.473,
50
+ "p90": 357.186,
51
+ "p95": 735.542,
52
+ "p99": 1038.228,
53
+ "avg": 165.443
54
+ },
55
+ "inter_token_latency_ms": {
56
+ "p50": 24.333,
57
+ "p60": 24.401,
58
+ "p70": 24.543,
59
+ "p80": 24.568,
60
+ "p90": 24.601,
61
+ "p95": 24.637,
62
+ "p99": 24.666,
63
+ "avg": 24.397
64
+ },
65
+ "failed_requests": 0,
66
+ "successful_requests": 8,
67
+ "request_rate": 0.2440110786188087,
68
+ "total_tokens_sent": 1600,
69
+ "e2e_latency_ms": {
70
+ "p50": 4585.648,
71
+ "p60": 4852.404,
72
+ "p70": 4955.58,
73
+ "p80": 5084.326,
74
+ "p90": 5491.26,
75
+ "p95": 5877.344,
76
+ "p99": 6186.212,
77
+ "avg": 4098.053
78
+ }
79
+ },
80
+ {
81
+ "id": "[email protected]/s",
82
+ "executor_type": "ConstantArrivalRate",
83
+ "config": {
84
+ "max_vus": 800,
85
+ "duration_secs": 120,
86
+ "rate": 1.0
87
+ },
88
+ "total_requests": 114,
89
+ "total_tokens": 21461,
90
+ "token_throughput_secs": 182.6075834319543,
91
+ "duration_ms": 117525,
92
+ "time_to_first_token_ms": {
93
+ "p50": 47.417,
94
+ "p60": 49.732,
95
+ "p70": 51.668,
96
+ "p80": 54.756,
97
+ "p90": 55.893,
98
+ "p95": 56.996,
99
+ "p99": 60.869,
100
+ "avg": 46.017
101
+ },
102
+ "inter_token_latency_ms": {
103
+ "p50": 25.057,
104
+ "p60": 25.1,
105
+ "p70": 25.173,
106
+ "p80": 25.235,
107
+ "p90": 25.33,
108
+ "p95": 25.43,
109
+ "p99": 25.473,
110
+ "avg": 25.111
111
+ },
112
+ "failed_requests": 0,
113
+ "successful_requests": 114,
114
+ "request_rate": 0.9700044038601552,
115
+ "total_tokens_sent": 22800,
116
+ "e2e_latency_ms": {
117
+ "p50": 4982.943,
118
+ "p60": 5042.008,
119
+ "p70": 5106.535,
120
+ "p80": 5157.482,
121
+ "p90": 5302.371,
122
+ "p95": 5419.485,
123
+ "p99": 5523.409,
124
+ "avg": 4752.857
125
+ }
126
+ },
127
+ {
128
+ "id": "[email protected]/s",
129
+ "executor_type": "ConstantArrivalRate",
130
+ "config": {
131
+ "max_vus": 800,
132
+ "duration_secs": 120,
133
+ "rate": 10.0
134
+ },
135
+ "total_requests": 1143,
136
+ "total_tokens": 217419,
137
+ "token_throughput_secs": 1816.6173506849534,
138
+ "duration_ms": 119683,
139
+ "time_to_first_token_ms": {
140
+ "p50": 57.522,
141
+ "p60": 60.424,
142
+ "p70": 63.326,
143
+ "p80": 66.071,
144
+ "p90": 68.799,
145
+ "p95": 69.978,
146
+ "p99": 72.022,
147
+ "avg": 57.702
148
+ },
149
+ "inter_token_latency_ms": {
150
+ "p50": 30.626,
151
+ "p60": 30.669,
152
+ "p70": 30.715,
153
+ "p80": 30.774,
154
+ "p90": 30.825,
155
+ "p95": 30.865,
156
+ "p99": 30.938,
157
+ "avg": 30.484
158
+ },
159
+ "failed_requests": 0,
160
+ "successful_requests": 1143,
161
+ "request_rate": 9.550194011714256,
162
+ "total_tokens_sent": 228600,
163
+ "e2e_latency_ms": {
164
+ "p50": 6057.161,
165
+ "p60": 6154.38,
166
+ "p70": 6229.34,
167
+ "p80": 6357.051,
168
+ "p90": 6486.978,
169
+ "p95": 6591.121,
170
+ "p99": 6755.652,
171
+ "avg": 5830.961
172
+ }
173
+ },
174
+ {
175
+ "id": "[email protected]/s",
176
+ "executor_type": "ConstantArrivalRate",
177
+ "config": {
178
+ "max_vus": 800,
179
+ "duration_secs": 120,
180
+ "rate": 30.0
181
+ },
182
+ "total_requests": 1810,
183
+ "total_tokens": 333423,
184
+ "token_throughput_secs": 2824.7053026343956,
185
+ "duration_ms": 118038,
186
+ "time_to_first_token_ms": {
187
+ "p50": 22932.695,
188
+ "p60": 24318.695,
189
+ "p70": 29936.767,
190
+ "p80": 31425.443,
191
+ "p90": 36187.994,
192
+ "p95": 37161.471,
193
+ "p99": 38279.808,
194
+ "avg": 19832.992
195
+ },
196
+ "inter_token_latency_ms": {
197
+ "p50": 66.633,
198
+ "p60": 67.904,
199
+ "p70": 69.306,
200
+ "p80": 73.383,
201
+ "p90": 92.178,
202
+ "p95": 99.967,
203
+ "p99": 159.213,
204
+ "avg": 71.51
205
+ },
206
+ "failed_requests": 0,
207
+ "successful_requests": 1810,
208
+ "request_rate": 15.334024940595748,
209
+ "total_tokens_sent": 362000,
210
+ "e2e_latency_ms": {
211
+ "p50": 33560.253,
212
+ "p60": 38196.989,
213
+ "p70": 43248.097,
214
+ "p80": 46500.801,
215
+ "p90": 48937.833,
216
+ "p95": 50024.293,
217
+ "p99": 51839.479,
218
+ "avg": 32606.281
219
+ }
220
+ },
221
+ {
222
+ "id": "[email protected]/s",
223
+ "executor_type": "ConstantArrivalRate",
224
+ "config": {
225
+ "max_vus": 800,
226
+ "duration_secs": 120,
227
+ "rate": 100.0
228
+ },
229
+ "total_requests": 1862,
230
+ "total_tokens": 343341,
231
+ "token_throughput_secs": 2926.455839084569,
232
+ "duration_ms": 117323,
233
+ "time_to_first_token_ms": {
234
+ "p50": 31244.139,
235
+ "p60": 33247.105,
236
+ "p70": 34817.178,
237
+ "p80": 37376.4,
238
+ "p90": 37949.506,
239
+ "p95": 38357.089,
240
+ "p99": 38609.173,
241
+ "avg": 26274.216
242
+ },
243
+ "inter_token_latency_ms": {
244
+ "p50": 66.847,
245
+ "p60": 68.552,
246
+ "p70": 71.272,
247
+ "p80": 86.491,
248
+ "p90": 99.169,
249
+ "p95": 105.417,
250
+ "p99": 169.433,
251
+ "avg": 73.816
252
+ },
253
+ "failed_requests": 0,
254
+ "successful_requests": 1862,
255
+ "request_rate": 15.870696399135168,
256
+ "total_tokens_sent": 372400,
257
+ "e2e_latency_ms": {
258
+ "p50": 45296.886,
259
+ "p60": 47352.112,
260
+ "p70": 48816.1,
261
+ "p80": 50047.695,
262
+ "p90": 51109.489,
263
+ "p95": 51997.385,
264
+ "p99": 53581.678,
265
+ "avg": 39393.313
266
+ }
267
+ }
268
+ ],
269
+ "start_time": "2025-05-27T14:14:31.942207084+00:00",
270
+ "end_time": "2025-05-27T14:24:12.744473375+00:00",
271
+ "system": {
272
+ "cpu": [
273
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu0@4699MHz",
274
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu1@4699MHz",
275
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu2@4699MHz",
276
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu3@4699MHz",
277
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu4@4699MHz",
278
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu5@4699MHz",
279
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu6@4699MHz",
280
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu7@4699MHz",
281
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu8@4699MHz",
282
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu9@4699MHz",
283
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu10@4699MHz",
284
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu11@4699MHz",
285
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu12@4699MHz",
286
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu13@4699MHz",
287
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu14@4699MHz",
288
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu15@4699MHz"
289
+ ],
290
+ "memory": "83.47 GB",
291
+ "os_name": "Debian GNU/Linux",
292
+ "os_version": "11",
293
+ "kernel": "5.15.167.4-microsoft-standard-WSL2",
294
+ "hostname": "computer"
295
+ }
296
+ }