experiment_name,backend.name,backend.version,backend.inter_op_num_threads,backend.intra_op_num_threads,backend._target_,backend.provider,backend.use_io_binding,backend.enable_profiling,backend.optimization,backend.optimization_config.optimization_level,backend.optimization_config.optimize_for_gpu,backend.optimization_config.fp16,backend.optimization_config.enable_transformers_specific_optimizations,backend.optimization_config.enable_gelu_approximation,backend.optimization_config.disable_gelu_fusion,backend.optimization_config.disable_layer_norm_fusion,backend.optimization_config.disable_attention_fusion,backend.optimization_config.disable_skip_layer_norm_fusion,backend.optimization_config.disable_bias_skip_layer_norm_fusion,backend.optimization_config.disable_bias_gelu_fusion,backend.optimization_config.use_mask_index,backend.optimization_config.no_attention_mask,backend.optimization_config.disable_embed_layer_norm_fusion,backend.optimization_config.disable_shape_inference,backend.optimization_config.use_multi_head_attention,backend.optimization_config.enable_gemm_fast_gelu_fusion,backend.optimization_config.use_raw_attention_mask,backend.optimization_config.disable_group_norm_fusion,backend.optimization_config.disable_packed_kv,backend.auto_optimization,backend.auto_optimization_config.for_gpu,backend.quantization,backend.quantization_config.is_static,backend.quantization_config.format,backend.quantization_config.mode,backend.quantization_config.activations_dtype,backend.quantization_config.activations_symmetric,backend.quantization_config.weights_dtype,backend.quantization_config.weights_symmetric,backend.quantization_config.per_channel,backend.quantization_config.reduce_range,backend.quantization_config.operators_to_quantize,backend.auto_quantization,backend.auto_quantization_config.is_static,backend.auto_quantization_config.per_channel,benchmark.name,benchmark._target_,benchmark.seed,benchmark.memory,benchmark.profile,benchmark.warmup_runs,benchmark.benchmark_duration,benchmark.batch_size,benchmark.new_tokens,model,device,task,environment.optimum_version,environment.transformers_version,environment.python_version,environment.system,environment.cpu,environment.cpu_count,environment.cpu_ram_mb,environment.gpu,environment.gpu_vram_mb,Unnamed: 0,forward.latency(s),forward.throughput(samples/s),generate.latency(s),generate.throughput(tokens/s),backend.auto_quantization_config.operators_to_quantize,backend.disable_grad,backend.eval_mode,backend.fp16,backend.bettertransformer,backend.torch_compile,forward.speedup(%),generate.speedup(%)
whisper_auto_opt(O1)+auto_qnt(avx512_vnni),onnxruntime,1.15.0,,,src.backend.onnxruntime.ORTBackend,CPUExecutionProvider,False,False,False,1,False,False,True,False,False,False,False,True,False,False,False,False,True,False,False,False,False,True,True,O1,False,False,False,QOperator,IntegerOps,QUInt8,False,QInt8,True,False,False,"['MatMul', 'Add']",avx512_vnni,False,False,inference,src.benchmark.inference.InferenceBenchmark,42,False,False,10,10,1,100,openai/whisper-base,cpu,automatic-speech-recognition,1.8.8.dev0,4.29.2,3.10.12,Linux, Intel(R) Xeon(R) CPU @ 2.20GHz,2,12982,CUDA not available,CUDA not available,0,1.68,0.594,4.88,20.5,"['Gather', 'Transpose', 'EmbedLayerNormalization', 'Attention', 'LSTM', 'ArgMax', 'Gemm', 'MatMul', 'Add', 'Mul', 'Relu', 'Clip', 'LeakyRelu', 'Sigmoid', 'MaxPool', 'GlobalAveragePool', 'Split', 'Pad', 'Reshape', 'Squeeze', 'Unsqueeze', 'Resize', 'AveragePool', 'Concat', 'Softmax', 'Where', 'ConvTranspose', 'InstanceNormalization']",,,,,,76.26112759643915,34.86842105263159
whisper_auto_opt(O3)+auto_qnt(avx512_vnni),onnxruntime,1.15.0,,,src.backend.onnxruntime.ORTBackend,CPUExecutionProvider,False,False,False,1,False,False,True,False,False,False,False,True,False,False,False,False,True,False,False,False,False,True,True,O3,False,False,False,QOperator,IntegerOps,QUInt8,False,QInt8,True,False,False,"['MatMul', 'Add']",avx512_vnni,False,False,inference,src.benchmark.inference.InferenceBenchmark,42,False,False,10,10,1,100,openai/whisper-base,cpu,automatic-speech-recognition,1.8.8.dev0,4.29.2,3.10.12,Linux, Intel(R) Xeon(R) CPU @ 2.20GHz,2,12982,CUDA not available,CUDA not available,0,1.7,0.588,4.34,23.0,"['Gather', 'Transpose', 'EmbedLayerNormalization', 'Attention', 'LSTM', 'ArgMax', 'Gemm', 'MatMul', 'Add', 'Mul', 'Relu', 'Clip', 'LeakyRelu', 'Sigmoid', 'MaxPool', 'GlobalAveragePool', 'Split', 'Pad', 'Reshape', 'Squeeze', 'Unsqueeze', 'Resize', 'AveragePool', 'Concat', 'Softmax', 'Where', 'ConvTranspose', 'InstanceNormalization']",,,,,,74.4807121661721,51.31578947368423
whisper_auto_qnt(arm64),onnxruntime,1.15.0,,,src.backend.onnxruntime.ORTBackend,CPUExecutionProvider,False,False,False,1,False,False,True,False,False,False,False,True,False,False,False,False,True,False,False,False,False,True,True,,False,False,False,QOperator,IntegerOps,QUInt8,False,QInt8,True,False,False,"['MatMul', 'Add']",arm64,False,False,inference,src.benchmark.inference.InferenceBenchmark,42,False,False,10,10,1,100,openai/whisper-base,cpu,automatic-speech-recognition,1.8.8.dev0,4.29.2,3.10.12,Linux, Intel(R) Xeon(R) CPU @ 2.20GHz,2,12982,CUDA not available,CUDA not available,0,1.72,0.581,4.36,22.9,,,,,,,72.4035608308605,50.6578947368421
whisper_auto_qnt(avx512),onnxruntime,1.15.0,,,src.backend.onnxruntime.ORTBackend,CPUExecutionProvider,False,False,False,1,False,False,True,False,False,False,False,True,False,False,False,False,True,False,False,False,False,True,True,,False,False,False,QOperator,IntegerOps,QUInt8,False,QInt8,True,False,False,"['MatMul', 'Add']",avx512,False,False,inference,src.benchmark.inference.InferenceBenchmark,42,False,False,10,10,1,100,openai/whisper-base,cpu,automatic-speech-recognition,1.8.8.dev0,4.29.2,3.10.12,Linux, Intel(R) Xeon(R) CPU @ 2.20GHz,2,12982,CUDA not available,CUDA not available,0,1.76,0.57,7.2,13.9,,,,,,,69.1394658753709,-8.552631578947366
whisper_auto_qnt(avx512_vnni),onnxruntime,1.15.0,,,src.backend.onnxruntime.ORTBackend,CPUExecutionProvider,False,False,False,1,False,False,True,False,False,False,False,True,False,False,False,False,True,False,False,False,False,True,True,,False,False,False,QOperator,IntegerOps,QUInt8,False,QInt8,True,False,False,"['MatMul', 'Add']",avx512_vnni,False,False,inference,src.benchmark.inference.InferenceBenchmark,42,False,False,10,10,1,100,openai/whisper-base,cpu,automatic-speech-recognition,1.8.8.dev0,4.29.2,3.10.12,Linux, Intel(R) Xeon(R) CPU @ 2.20GHz,2,12982,CUDA not available,CUDA not available,0,1.78,0.562,4.74,21.1,,,,,,,66.76557863501485,38.81578947368423
whisper_auto_opt(O2)+auto_qnt(avx512),onnxruntime,1.15.0,,,src.backend.onnxruntime.ORTBackend,CPUExecutionProvider,False,False,False,1,False,False,True,False,False,False,False,True,False,False,False,False,True,False,False,False,False,True,True,O2,False,False,False,QOperator,IntegerOps,QUInt8,False,QInt8,True,False,False,"['MatMul', 'Add']",avx512,False,False,inference,src.benchmark.inference.InferenceBenchmark,42,False,False,10,10,1,100,openai/whisper-base,cpu,automatic-speech-recognition,1.8.8.dev0,4.29.2,3.10.12,Linux, Intel(R) Xeon(R) CPU @ 2.20GHz,2,12982,CUDA not available,CUDA not available,0,1.88,0.531,4.54,22.0,"['Gather', 'Transpose', 'EmbedLayerNormalization', 'Attention', 'LSTM', 'ArgMax', 'Gemm', 'MatMul', 'Add', 'Mul', 'Relu', 'Clip', 'LeakyRelu', 'Sigmoid', 'MaxPool', 'GlobalAveragePool', 'Split', 'Pad', 'Reshape', 'Squeeze', 'Unsqueeze', 'Resize', 'AveragePool', 'Concat', 'Softmax', 'Where', 'ConvTranspose', 'InstanceNormalization']",,,,,,57.56676557863501,44.736842105263165
whisper_auto_opt(O3)+auto_qnt(arm64),onnxruntime,1.15.0,,,src.backend.onnxruntime.ORTBackend,CPUExecutionProvider,False,False,False,1,False,False,True,False,False,False,False,True,False,False,False,False,True,False,False,False,False,True,True,O3,False,False,False,QOperator,IntegerOps,QUInt8,False,QInt8,True,False,False,"['MatMul', 'Add']",arm64,False,False,inference,src.benchmark.inference.InferenceBenchmark,42,False,False,10,10,1,100,openai/whisper-base,cpu,automatic-speech-recognition,1.8.8.dev0,4.29.2,3.10.12,Linux, Intel(R) Xeon(R) CPU @ 2.20GHz,2,12982,CUDA not available,CUDA not available,0,1.92,0.522,4.38,22.8,"['Gather', 'Transpose', 'EmbedLayerNormalization', 'Attention', 'LSTM', 'ArgMax', 'Gemm', 'MatMul', 'Add', 'Mul', 'Relu', 'Clip', 'LeakyRelu', 'Sigmoid', 'MaxPool', 'GlobalAveragePool', 'Split', 'Pad', 'Reshape', 'Squeeze', 'Unsqueeze', 'Resize', 'AveragePool', 'Concat', 'Softmax', 'Where', 'ConvTranspose', 'InstanceNormalization']",,,,,,54.89614243323442,50.00000000000002
whisper_auto_opt(O2)+auto_qnt(avx512_vnni),onnxruntime,1.15.0,,,src.backend.onnxruntime.ORTBackend,CPUExecutionProvider,False,False,False,1,False,False,True,False,False,False,False,True,False,False,False,False,True,False,False,False,False,True,True,O2,False,False,False,QOperator,IntegerOps,QUInt8,False,QInt8,True,False,False,"['MatMul', 'Add']",avx512_vnni,False,False,inference,src.benchmark.inference.InferenceBenchmark,42,False,False,10,10,1,100,openai/whisper-base,cpu,automatic-speech-recognition,1.8.8.dev0,4.29.2,3.10.12,Linux, Intel(R) Xeon(R) CPU @ 2.20GHz,2,12982,CUDA not available,CUDA not available,0,2.01,0.498,4.49,22.3,"['Gather', 'Transpose', 'EmbedLayerNormalization', 'Attention', 'LSTM', 'ArgMax', 'Gemm', 'MatMul', 'Add', 'Mul', 'Relu', 'Clip', 'LeakyRelu', 'Sigmoid', 'MaxPool', 'GlobalAveragePool', 'Split', 'Pad', 'Reshape', 'Squeeze', 'Unsqueeze', 'Resize', 'AveragePool', 'Concat', 'Softmax', 'Where', 'ConvTranspose', 'InstanceNormalization']",,,,,,47.77448071216617,46.710526315789494
whisper_auto_opt(O3)+auto_qnt(avx2),onnxruntime,1.15.0,,,src.backend.onnxruntime.ORTBackend,CPUExecutionProvider,False,False,False,1,False,False,True,False,False,False,False,True,False,False,False,False,True,False,False,False,False,True,True,O3,False,False,False,QOperator,IntegerOps,QUInt8,False,QInt8,True,False,False,"['MatMul', 'Add']",avx2,False,False,inference,src.benchmark.inference.InferenceBenchmark,42,False,False,10,10,1,100,openai/whisper-base,cpu,automatic-speech-recognition,1.8.8.dev0,4.29.2,3.10.12,Linux, Intel(R) Xeon(R) CPU @ 2.20GHz,2,12982,CUDA not available,CUDA not available,0,2.02,0.496,4.73,21.1,"['Gather', 'Transpose', 'EmbedLayerNormalization', 'Attention', 'LSTM', 'ArgMax', 'Gemm', 'MatMul', 'Add', 'Mul', 'Relu', 'Clip', 'LeakyRelu', 'Sigmoid', 'MaxPool', 'GlobalAveragePool', 'Split', 'Pad', 'Reshape', 'Squeeze', 'Unsqueeze', 'Resize', 'AveragePool', 'Concat', 'Softmax', 'Where', 'ConvTranspose', 'InstanceNormalization']",,,,,,47.181008902077146,38.81578947368423
whisper_auto_opt(O3)+auto_qnt(avx512),onnxruntime,1.15.0,,,src.backend.onnxruntime.ORTBackend,CPUExecutionProvider,False,False,False,1,False,False,True,False,False,False,False,True,False,False,False,False,True,False,False,False,False,True,True,O3,False,False,False,QOperator,IntegerOps,QUInt8,False,QInt8,True,False,False,"['MatMul', 'Add']",avx512,False,False,inference,src.benchmark.inference.InferenceBenchmark,42,False,False,10,10,1,100,openai/whisper-base,cpu,automatic-speech-recognition,1.8.8.dev0,4.29.2,3.10.12,Linux, Intel(R) Xeon(R) CPU @ 2.20GHz,2,12982,CUDA not available,CUDA not available,0,2.02,0.495,4.36,22.9,"['Gather', 'Transpose', 'EmbedLayerNormalization', 'Attention', 'LSTM', 'ArgMax', 'Gemm', 'MatMul', 'Add', 'Mul', 'Relu', 'Clip', 'LeakyRelu', 'Sigmoid', 'MaxPool', 'GlobalAveragePool', 'Split', 'Pad', 'Reshape', 'Squeeze', 'Unsqueeze', 'Resize', 'AveragePool', 'Concat', 'Softmax', 'Where', 'ConvTranspose', 'InstanceNormalization']",,,,,,46.88427299703264,50.6578947368421
whisper_auto_opt(O1)+auto_qnt(avx512),onnxruntime,1.15.0,,,src.backend.onnxruntime.ORTBackend,CPUExecutionProvider,False,False,False,1,False,False,True,False,False,False,False,True,False,False,False,False,True,False,False,False,False,True,True,O1,False,False,False,QOperator,IntegerOps,QUInt8,False,QInt8,True,False,False,"['MatMul', 'Add']",avx512,False,False,inference,src.benchmark.inference.InferenceBenchmark,42,False,False,10,10,1,100,openai/whisper-base,cpu,automatic-speech-recognition,1.8.8.dev0,4.29.2,3.10.12,Linux, Intel(R) Xeon(R) CPU @ 2.20GHz,2,12982,CUDA not available,CUDA not available,0,2.03,0.493,4.86,20.6,"['Gather', 'Transpose', 'EmbedLayerNormalization', 'Attention', 'LSTM', 'ArgMax', 'Gemm', 'MatMul', 'Add', 'Mul', 'Relu', 'Clip', 'LeakyRelu', 'Sigmoid', 'MaxPool', 'GlobalAveragePool', 'Split', 'Pad', 'Reshape', 'Squeeze', 'Unsqueeze', 'Resize', 'AveragePool', 'Concat', 'Softmax', 'Where', 'ConvTranspose', 'InstanceNormalization']",,,,,,46.29080118694362,35.52631578947369
whisper_auto_qnt(avx2),onnxruntime,1.15.0,,,src.backend.onnxruntime.ORTBackend,CPUExecutionProvider,False,False,False,1,False,False,True,False,False,False,False,True,False,False,False,False,True,False,False,False,False,True,True,,False,False,False,QOperator,IntegerOps,QUInt8,False,QInt8,True,False,False,"['MatMul', 'Add']",avx2,False,False,inference,src.benchmark.inference.InferenceBenchmark,42,False,False,10,10,1,100,openai/whisper-base,cpu,automatic-speech-recognition,1.8.8.dev0,4.29.2,3.10.12,Linux, Intel(R) Xeon(R) CPU @ 2.20GHz,2,12982,CUDA not available,CUDA not available,0,2.05,0.489,4.74,21.1,,,,,,,45.10385756676556,38.81578947368423
whisper_auto_opt(O1)+auto_qnt(avx2),onnxruntime,1.15.0,,,src.backend.onnxruntime.ORTBackend,CPUExecutionProvider,False,False,False,1,False,False,True,False,False,False,False,True,False,False,False,False,True,False,False,False,False,True,True,O1,False,False,False,QOperator,IntegerOps,QUInt8,False,QInt8,True,False,False,"['MatMul', 'Add']",avx2,False,False,inference,src.benchmark.inference.InferenceBenchmark,42,False,False,10,10,1,100,openai/whisper-base,cpu,automatic-speech-recognition,1.8.8.dev0,4.29.2,3.10.12,Linux, Intel(R) Xeon(R) CPU @ 2.20GHz,2,12982,CUDA not available,CUDA not available,0,2.18,0.46,6.07,16.5,"['Gather', 'Transpose', 'EmbedLayerNormalization', 'Attention', 'LSTM', 'ArgMax', 'Gemm', 'MatMul', 'Add', 'Mul', 'Relu', 'Clip', 'LeakyRelu', 'Sigmoid', 'MaxPool', 'GlobalAveragePool', 'Split', 'Pad', 'Reshape', 'Squeeze', 'Unsqueeze', 'Resize', 'AveragePool', 'Concat', 'Softmax', 'Where', 'ConvTranspose', 'InstanceNormalization']",,,,,,36.498516320474785,8.552631578947366
whisper_auto_opt(O2)+auto_qnt(arm64),onnxruntime,1.15.0,,,src.backend.onnxruntime.ORTBackend,CPUExecutionProvider,False,False,False,1,False,False,True,False,False,False,False,True,False,False,False,False,True,False,False,False,False,True,True,O2,False,False,False,QOperator,IntegerOps,QUInt8,False,QInt8,True,False,False,"['MatMul', 'Add']",arm64,False,False,inference,src.benchmark.inference.InferenceBenchmark,42,False,False,10,10,1,100,openai/whisper-base,cpu,automatic-speech-recognition,1.8.8.dev0,4.29.2,3.10.12,Linux, Intel(R) Xeon(R) CPU @ 2.20GHz,2,12982,CUDA not available,CUDA not available,0,2.23,0.449,5.19,19.3,"['Gather', 'Transpose', 'EmbedLayerNormalization', 'Attention', 'LSTM', 'ArgMax', 'Gemm', 'MatMul', 'Add', 'Mul', 'Relu', 'Clip', 'LeakyRelu', 'Sigmoid', 'MaxPool', 'GlobalAveragePool', 'Split', 'Pad', 'Reshape', 'Squeeze', 'Unsqueeze', 'Resize', 'AveragePool', 'Concat', 'Softmax', 'Where', 'ConvTranspose', 'InstanceNormalization']",,,,,,33.23442136498515,26.97368421052633
whisper_auto_opt(O1)+auto_qnt(arm64),onnxruntime,1.15.0,,,src.backend.onnxruntime.ORTBackend,CPUExecutionProvider,False,False,False,1,False,False,True,False,False,False,False,True,False,False,False,False,True,False,False,False,False,True,True,O1,False,False,False,QOperator,IntegerOps,QUInt8,False,QInt8,True,False,False,"['MatMul', 'Add']",arm64,False,False,inference,src.benchmark.inference.InferenceBenchmark,42,False,False,10,10,1,100,openai/whisper-base,cpu,automatic-speech-recognition,1.8.8.dev0,4.29.2,3.10.12,Linux, Intel(R) Xeon(R) CPU @ 2.20GHz,2,12982,CUDA not available,CUDA not available,0,2.24,0.447,4.57,21.9,"['Gather', 'Transpose', 'EmbedLayerNormalization', 'Attention', 'LSTM', 'ArgMax', 'Gemm', 'MatMul', 'Add', 'Mul', 'Relu', 'Clip', 'LeakyRelu', 'Sigmoid', 'MaxPool', 'GlobalAveragePool', 'Split', 'Pad', 'Reshape', 'Squeeze', 'Unsqueeze', 'Resize', 'AveragePool', 'Concat', 'Softmax', 'Where', 'ConvTranspose', 'InstanceNormalization']",,,,,,32.64094955489614,44.07894736842104
whisper_auto_opt(O2)+auto_qnt(avx2),onnxruntime,1.15.0,,,src.backend.onnxruntime.ORTBackend,CPUExecutionProvider,False,False,False,1,False,False,True,False,False,False,False,True,False,False,False,False,True,False,False,False,False,True,True,O2,False,False,False,QOperator,IntegerOps,QUInt8,False,QInt8,True,False,False,"['MatMul', 'Add']",avx2,False,False,inference,src.benchmark.inference.InferenceBenchmark,42,False,False,10,10,1,100,openai/whisper-base,cpu,automatic-speech-recognition,1.8.8.dev0,4.29.2,3.10.12,Linux, Intel(R) Xeon(R) CPU @ 2.20GHz,2,12982,CUDA not available,CUDA not available,0,2.3,0.435,5.61,17.8,"['Gather', 'Transpose', 'EmbedLayerNormalization', 'Attention', 'LSTM', 'ArgMax', 'Gemm', 'MatMul', 'Add', 'Mul', 'Relu', 'Clip', 'LeakyRelu', 'Sigmoid', 'MaxPool', 'GlobalAveragePool', 'Split', 'Pad', 'Reshape', 'Squeeze', 'Unsqueeze', 'Resize', 'AveragePool', 'Concat', 'Softmax', 'Where', 'ConvTranspose', 'InstanceNormalization']",,,,,,29.080118694362,17.105263157894758
whisper_baseline,pytorch,2.0.1+cu118,,,src.backend.pytorch.PyTorchBackend,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,inference,src.benchmark.inference.InferenceBenchmark,42,False,False,10,10,1,100,openai/whisper-base,cpu,automatic-speech-recognition,1.8.8.dev0,4.29.2,3.10.12,Linux, Intel(R) Xeon(R) CPU @ 2.20GHz,2,12982,CUDA not available,CUDA not available,0,2.97,0.337,6.57,15.2,,False,False,False,False,False,0.0,0.0
