| { | |
| "current_gpu_type": "NVIDIA A10G", | |
| "current_gpu_total_memory": 22716.5, | |
| "memory_inference_first": 9396.0, | |
| "memory_inference": 9396.0, | |
| "token_generation_latency_sync": 41.4751594543457, | |
| "token_generation_latency_async": 41.440220922231674, | |
| "token_generation_throughput_sync": 0.024110817490666007, | |
| "token_generation_throughput_async": 0.024131145484881433, | |
| "token_generation_CO2_emissions": 2.7722372785106468e-06, | |
| "token_generation_energy_consumption": 0.0013460853224844924, | |
| "inference_latency_sync": 34.25094375610352, | |
| "inference_latency_async": 27.751636505126953, | |
| "inference_throughput_sync": 0.02919627579084737, | |
| "inference_throughput_async": 0.036033911002518924, | |
| "inference_CO2_emissions": 1.7203736862892723e-06, | |
| "inference_energy_consumption": 9.17469136023194e-06 | |
| } |