experiment_name,backend.name,backend.version,backend.inter_op_num_threads,backend.intra_op_num_threads,backend._target_,backend.provider,backend.use_io_binding,backend.enable_profiling,backend.optimization,backend.optimization_config.optimization_level,backend.optimization_config.optimize_for_gpu,backend.optimization_config.fp16,backend.optimization_config.enable_transformers_specific_optimizations,backend.optimization_config.enable_gelu_approximation,backend.optimization_config.disable_gelu_fusion,backend.optimization_config.disable_layer_norm_fusion,backend.optimization_config.disable_attention_fusion,backend.optimization_config.disable_skip_layer_norm_fusion,backend.optimization_config.disable_bias_skip_layer_norm_fusion,backend.optimization_config.disable_bias_gelu_fusion,backend.optimization_config.use_mask_index,backend.optimization_config.no_attention_mask,backend.optimization_config.disable_embed_layer_norm_fusion,backend.optimization_config.disable_shape_inference,backend.optimization_config.use_multi_head_attention,backend.optimization_config.enable_gemm_fast_gelu_fusion,backend.optimization_config.use_raw_attention_mask,backend.optimization_config.disable_group_norm_fusion,backend.optimization_config.disable_packed_kv,backend.auto_optimization,backend.auto_optimization_config.for_gpu,backend.quantization,backend.quantization_config.is_static,backend.quantization_config.format,backend.quantization_config.mode,backend.quantization_config.activations_dtype,backend.quantization_config.activations_symmetric,backend.quantization_config.weights_dtype,backend.quantization_config.weights_symmetric,backend.quantization_config.per_channel,backend.quantization_config.reduce_range,backend.quantization_config.operators_to_quantize,backend.auto_quantization,backend.auto_quantization_config.is_static,backend.auto_quantization_config.per_channel,benchmark.name,benchmark._target_,benchmark.seed,benchmark.memory,benchmark.profile,benchmark.warmup_runs,benchmark.benchmark_duration,benchmark.batch_size,benchmark.new_tokens,model,device,task,environment.optimum_version,environment.transformers_version,environment.python_version,environment.system,environment.cpu,environment.cpu_count,environment.cpu_ram_mb,environment.gpu,environment.gpu_vram_mb,Unnamed: 0,forward.latency(s),forward.throughput(samples/s),generate.latency(s),generate.throughput(tokens/s),backend.auto_quantization_config.operators_to_quantize,backend.disable_grad,backend.eval_mode,backend.fp16,backend.bettertransformer,backend.torch_compile,forward.speedup(%),generate.speedup(%)
whisper_auto_opt(O1)+auto_qnt(arm64),onnxruntime,1.15.0,,,src.backend.onnxruntime.ORTBackend,CPUExecutionProvider,False,False,False,1,False,False,True,False,False,False,False,True,False,False,False,False,True,False,False,False,False,True,True,O1,False,False,False,QOperator,IntegerOps,QUInt8,False,QInt8,True,False,False,"['MatMul', 'Add']",arm64,False,False,inference,src.benchmark.inference.InferenceBenchmark,42,False,False,10,10,1,10,openai/whisper-base,cpu,automatic-speech-recognition,1.8.8.dev0,4.29.2,3.10.12,Linux, Intel(R) Xeon(R) CPU @ 2.20GHz,2,12982,CUDA not available,CUDA not available,0,1.67,0.598,1.79,5.58,"['Gather', 'Transpose', 'EmbedLayerNormalization', 'Attention', 'LSTM', 'ArgMax', 'Gemm', 'MatMul', 'Add', 'Mul', 'Relu', 'Clip', 'LeakyRelu', 'Sigmoid', 'MaxPool', 'GlobalAveragePool', 'Split', 'Pad', 'Reshape', 'Squeeze', 'Unsqueeze', 'Resize', 'AveragePool', 'Concat', 'Softmax', 'Where', 'ConvTranspose', 'InstanceNormalization']",,,,,,58.20105820105819,68.58006042296071
whisper_auto_opt(O2)+auto_qnt(avx512_vnni),onnxruntime,1.15.0,,,src.backend.onnxruntime.ORTBackend,CPUExecutionProvider,False,False,False,1,False,False,True,False,False,False,False,True,False,False,False,False,True,False,False,False,False,True,True,O2,False,False,False,QOperator,IntegerOps,QUInt8,False,QInt8,True,False,False,"['MatMul', 'Add']",avx512_vnni,False,False,inference,src.benchmark.inference.InferenceBenchmark,42,False,False,10,10,1,10,openai/whisper-base,cpu,automatic-speech-recognition,1.8.8.dev0,4.29.2,3.10.12,Linux, Intel(R) Xeon(R) CPU @ 2.20GHz,2,12982,CUDA not available,CUDA not available,0,1.68,0.595,2.18,4.59,"['Gather', 'Transpose', 'EmbedLayerNormalization', 'Attention', 'LSTM', 'ArgMax', 'Gemm', 'MatMul', 'Add', 'Mul', 'Relu', 'Clip', 'LeakyRelu', 'Sigmoid', 'MaxPool', 'GlobalAveragePool', 'Split', 'Pad', 'Reshape', 'Squeeze', 'Unsqueeze', 'Resize', 'AveragePool', 'Concat', 'Softmax', 'Where', 'ConvTranspose', 'InstanceNormalization']",,,,,,57.4074074074074,38.67069486404833
whisper_auto_opt(O2)+auto_qnt(avx512),onnxruntime,1.15.0,,,src.backend.onnxruntime.ORTBackend,CPUExecutionProvider,False,False,False,1,False,False,True,False,False,False,False,True,False,False,False,False,True,False,False,False,False,True,True,O2,False,False,False,QOperator,IntegerOps,QUInt8,False,QInt8,True,False,False,"['MatMul', 'Add']",avx512,False,False,inference,src.benchmark.inference.InferenceBenchmark,42,False,False,10,10,1,10,openai/whisper-base,cpu,automatic-speech-recognition,1.8.8.dev0,4.29.2,3.10.12,Linux, Intel(R) Xeon(R) CPU @ 2.20GHz,2,12982,CUDA not available,CUDA not available,0,1.7,0.589,1.76,5.67,"['Gather', 'Transpose', 'EmbedLayerNormalization', 'Attention', 'LSTM', 'ArgMax', 'Gemm', 'MatMul', 'Add', 'Mul', 'Relu', 'Clip', 'LeakyRelu', 'Sigmoid', 'MaxPool', 'GlobalAveragePool', 'Split', 'Pad', 'Reshape', 'Squeeze', 'Unsqueeze', 'Resize', 'AveragePool', 'Concat', 'Softmax', 'Where', 'ConvTranspose', 'InstanceNormalization']",,,,,,55.820105820105816,71.29909365558913
whisper_auto_opt(O3)+auto_qnt(arm64),onnxruntime,1.15.0,,,src.backend.onnxruntime.ORTBackend,CPUExecutionProvider,False,False,False,1,False,False,True,False,False,False,False,True,False,False,False,False,True,False,False,False,False,True,True,O3,False,False,False,QOperator,IntegerOps,QUInt8,False,QInt8,True,False,False,"['MatMul', 'Add']",arm64,False,False,inference,src.benchmark.inference.InferenceBenchmark,42,False,False,10,10,1,10,openai/whisper-base,cpu,automatic-speech-recognition,1.8.8.dev0,4.29.2,3.10.12,Linux, Intel(R) Xeon(R) CPU @ 2.20GHz,2,12982,CUDA not available,CUDA not available,0,1.75,0.573,1.81,5.54,"['Gather', 'Transpose', 'EmbedLayerNormalization', 'Attention', 'LSTM', 'ArgMax', 'Gemm', 'MatMul', 'Add', 'Mul', 'Relu', 'Clip', 'LeakyRelu', 'Sigmoid', 'MaxPool', 'GlobalAveragePool', 'Split', 'Pad', 'Reshape', 'Squeeze', 'Unsqueeze', 'Resize', 'AveragePool', 'Concat', 'Softmax', 'Where', 'ConvTranspose', 'InstanceNormalization']",,,,,,51.58730158730158,67.37160120845923
whisper_auto_opt(O3)+auto_qnt(avx512),onnxruntime,1.15.0,,,src.backend.onnxruntime.ORTBackend,CPUExecutionProvider,False,False,False,1,False,False,True,False,False,False,False,True,False,False,False,False,True,False,False,False,False,True,True,O3,False,False,False,QOperator,IntegerOps,QUInt8,False,QInt8,True,False,False,"['MatMul', 'Add']",avx512,False,False,inference,src.benchmark.inference.InferenceBenchmark,42,False,False,10,10,1,10,openai/whisper-base,cpu,automatic-speech-recognition,1.8.8.dev0,4.29.2,3.10.12,Linux, Intel(R) Xeon(R) CPU @ 2.20GHz,2,12982,CUDA not available,CUDA not available,0,1.86,0.538,1.99,5.03,"['Gather', 'Transpose', 'EmbedLayerNormalization', 'Attention', 'LSTM', 'ArgMax', 'Gemm', 'MatMul', 'Add', 'Mul', 'Relu', 'Clip', 'LeakyRelu', 'Sigmoid', 'MaxPool', 'GlobalAveragePool', 'Split', 'Pad', 'Reshape', 'Squeeze', 'Unsqueeze', 'Resize', 'AveragePool', 'Concat', 'Softmax', 'Where', 'ConvTranspose', 'InstanceNormalization']",,,,,,42.32804232804233,51.96374622356497
whisper_auto_qnt(avx2),onnxruntime,1.15.0,,,src.backend.onnxruntime.ORTBackend,CPUExecutionProvider,False,False,False,1,False,False,True,False,False,False,False,True,False,False,False,False,True,False,False,False,False,True,True,,False,False,False,QOperator,IntegerOps,QUInt8,False,QInt8,True,False,False,"['MatMul', 'Add']",avx2,False,False,inference,src.benchmark.inference.InferenceBenchmark,42,False,False,10,10,1,10,openai/whisper-base,cpu,automatic-speech-recognition,1.8.8.dev0,4.29.2,3.10.12,Linux, Intel(R) Xeon(R) CPU @ 2.20GHz,2,12982,CUDA not available,CUDA not available,0,1.86,0.538,1.95,5.12,,,,,,,42.32804232804233,54.68277945619335
whisper_auto_qnt(avx512),onnxruntime,1.15.0,,,src.backend.onnxruntime.ORTBackend,CPUExecutionProvider,False,False,False,1,False,False,True,False,False,False,False,True,False,False,False,False,True,False,False,False,False,True,True,,False,False,False,QOperator,IntegerOps,QUInt8,False,QInt8,True,False,False,"['MatMul', 'Add']",avx512,False,False,inference,src.benchmark.inference.InferenceBenchmark,42,False,False,10,10,1,10,openai/whisper-base,cpu,automatic-speech-recognition,1.8.8.dev0,4.29.2,3.10.12,Linux, Intel(R) Xeon(R) CPU @ 2.20GHz,2,12982,CUDA not available,CUDA not available,0,1.93,0.519,2.03,4.92,,,,,,,37.301587301587304,48.640483383685805
whisper_auto_opt(O3)+auto_qnt(avx512_vnni),onnxruntime,1.15.0,,,src.backend.onnxruntime.ORTBackend,CPUExecutionProvider,False,False,False,1,False,False,True,False,False,False,False,True,False,False,False,False,True,False,False,False,False,True,True,O3,False,False,False,QOperator,IntegerOps,QUInt8,False,QInt8,True,False,False,"['MatMul', 'Add']",avx512_vnni,False,False,inference,src.benchmark.inference.InferenceBenchmark,42,False,False,10,10,1,10,openai/whisper-base,cpu,automatic-speech-recognition,1.8.8.dev0,4.29.2,3.10.12,Linux, Intel(R) Xeon(R) CPU @ 2.20GHz,2,12982,CUDA not available,CUDA not available,0,1.95,0.514,2.06,4.84,"['Gather', 'Transpose', 'EmbedLayerNormalization', 'Attention', 'LSTM', 'ArgMax', 'Gemm', 'MatMul', 'Add', 'Mul', 'Relu', 'Clip', 'LeakyRelu', 'Sigmoid', 'MaxPool', 'GlobalAveragePool', 'Split', 'Pad', 'Reshape', 'Squeeze', 'Unsqueeze', 'Resize', 'AveragePool', 'Concat', 'Softmax', 'Where', 'ConvTranspose', 'InstanceNormalization']",,,,,,35.978835978835974,46.223564954682786
whisper_auto_qnt(arm64),onnxruntime,1.15.0,,,src.backend.onnxruntime.ORTBackend,CPUExecutionProvider,False,False,False,1,False,False,True,False,False,False,False,True,False,False,False,False,True,False,False,False,False,True,True,,False,False,False,QOperator,IntegerOps,QUInt8,False,QInt8,True,False,False,"['MatMul', 'Add']",arm64,False,False,inference,src.benchmark.inference.InferenceBenchmark,42,False,False,10,10,1,10,openai/whisper-base,cpu,automatic-speech-recognition,1.8.8.dev0,4.29.2,3.10.12,Linux, Intel(R) Xeon(R) CPU @ 2.20GHz,2,12982,CUDA not available,CUDA not available,0,2.05,0.488,2.04,4.9,,,,,,,29.100529100529094,48.03625377643505
whisper_auto_opt(O2)+auto_qnt(arm64),onnxruntime,1.15.0,,,src.backend.onnxruntime.ORTBackend,CPUExecutionProvider,False,False,False,1,False,False,True,False,False,False,False,True,False,False,False,False,True,False,False,False,False,True,True,O2,False,False,False,QOperator,IntegerOps,QUInt8,False,QInt8,True,False,False,"['MatMul', 'Add']",arm64,False,False,inference,src.benchmark.inference.InferenceBenchmark,42,False,False,10,10,1,10,openai/whisper-base,cpu,automatic-speech-recognition,1.8.8.dev0,4.29.2,3.10.12,Linux, Intel(R) Xeon(R) CPU @ 2.20GHz,2,12982,CUDA not available,CUDA not available,0,2.05,0.487,2.25,4.44,"['Gather', 'Transpose', 'EmbedLayerNormalization', 'Attention', 'LSTM', 'ArgMax', 'Gemm', 'MatMul', 'Add', 'Mul', 'Relu', 'Clip', 'LeakyRelu', 'Sigmoid', 'MaxPool', 'GlobalAveragePool', 'Split', 'Pad', 'Reshape', 'Squeeze', 'Unsqueeze', 'Resize', 'AveragePool', 'Concat', 'Softmax', 'Where', 'ConvTranspose', 'InstanceNormalization']",,,,,,28.835978835978835,34.13897280966769
whisper_auto_qnt(avx512_vnni),onnxruntime,1.15.0,,,src.backend.onnxruntime.ORTBackend,CPUExecutionProvider,False,False,False,1,False,False,True,False,False,False,False,True,False,False,False,False,True,False,False,False,False,True,True,,False,False,False,QOperator,IntegerOps,QUInt8,False,QInt8,True,False,False,"['MatMul', 'Add']",avx512_vnni,False,False,inference,src.benchmark.inference.InferenceBenchmark,42,False,False,10,10,1,10,openai/whisper-base,cpu,automatic-speech-recognition,1.8.8.dev0,4.29.2,3.10.12,Linux, Intel(R) Xeon(R) CPU @ 2.20GHz,2,12982,CUDA not available,CUDA not available,0,2.06,0.485,2.06,4.85,,,,,,,28.306878306878303,46.525679758308144
whisper_auto_opt(O2)+auto_qnt(avx2),onnxruntime,1.15.0,,,src.backend.onnxruntime.ORTBackend,CPUExecutionProvider,False,False,False,1,False,False,True,False,False,False,False,True,False,False,False,False,True,False,False,False,False,True,True,O2,False,False,False,QOperator,IntegerOps,QUInt8,False,QInt8,True,False,False,"['MatMul', 'Add']",avx2,False,False,inference,src.benchmark.inference.InferenceBenchmark,42,False,False,10,10,1,10,openai/whisper-base,cpu,automatic-speech-recognition,1.8.8.dev0,4.29.2,3.10.12,Linux, Intel(R) Xeon(R) CPU @ 2.20GHz,2,12982,CUDA not available,CUDA not available,0,2.1,0.476,2.56,3.91,"['Gather', 'Transpose', 'EmbedLayerNormalization', 'Attention', 'LSTM', 'ArgMax', 'Gemm', 'MatMul', 'Add', 'Mul', 'Relu', 'Clip', 'LeakyRelu', 'Sigmoid', 'MaxPool', 'GlobalAveragePool', 'Split', 'Pad', 'Reshape', 'Squeeze', 'Unsqueeze', 'Resize', 'AveragePool', 'Concat', 'Softmax', 'Where', 'ConvTranspose', 'InstanceNormalization']",,,,,,25.92592592592593,18.126888217522662
whisper_auto_opt(O1)+auto_qnt(avx512),onnxruntime,1.15.0,,,src.backend.onnxruntime.ORTBackend,CPUExecutionProvider,False,False,False,1,False,False,True,False,False,False,False,True,False,False,False,False,True,False,False,False,False,True,True,O1,False,False,False,QOperator,IntegerOps,QUInt8,False,QInt8,True,False,False,"['MatMul', 'Add']",avx512,False,False,inference,src.benchmark.inference.InferenceBenchmark,42,False,False,10,10,1,10,openai/whisper-base,cpu,automatic-speech-recognition,1.8.8.dev0,4.29.2,3.10.12,Linux, Intel(R) Xeon(R) CPU @ 2.20GHz,2,12982,CUDA not available,CUDA not available,0,2.11,0.474,2.35,4.25,"['Gather', 'Transpose', 'EmbedLayerNormalization', 'Attention', 'LSTM', 'ArgMax', 'Gemm', 'MatMul', 'Add', 'Mul', 'Relu', 'Clip', 'LeakyRelu', 'Sigmoid', 'MaxPool', 'GlobalAveragePool', 'Split', 'Pad', 'Reshape', 'Squeeze', 'Unsqueeze', 'Resize', 'AveragePool', 'Concat', 'Softmax', 'Where', 'ConvTranspose', 'InstanceNormalization']",,,,,,25.396825396825395,28.398791540785506
whisper_auto_opt(O1)+auto_qnt(avx512_vnni),onnxruntime,1.15.0,,,src.backend.onnxruntime.ORTBackend,CPUExecutionProvider,False,False,False,1,False,False,True,False,False,False,False,True,False,False,False,False,True,False,False,False,False,True,True,O1,False,False,False,QOperator,IntegerOps,QUInt8,False,QInt8,True,False,False,"['MatMul', 'Add']",avx512_vnni,False,False,inference,src.benchmark.inference.InferenceBenchmark,42,False,False,10,10,1,10,openai/whisper-base,cpu,automatic-speech-recognition,1.8.8.dev0,4.29.2,3.10.12,Linux, Intel(R) Xeon(R) CPU @ 2.20GHz,2,12982,CUDA not available,CUDA not available,0,2.13,0.469,2.19,4.56,"['Gather', 'Transpose', 'EmbedLayerNormalization', 'Attention', 'LSTM', 'ArgMax', 'Gemm', 'MatMul', 'Add', 'Mul', 'Relu', 'Clip', 'LeakyRelu', 'Sigmoid', 'MaxPool', 'GlobalAveragePool', 'Split', 'Pad', 'Reshape', 'Squeeze', 'Unsqueeze', 'Resize', 'AveragePool', 'Concat', 'Softmax', 'Where', 'ConvTranspose', 'InstanceNormalization']",,,,,,24.07407407407407,37.76435045317219
whisper_auto_opt(O3)+auto_qnt(avx2),onnxruntime,1.15.0,,,src.backend.onnxruntime.ORTBackend,CPUExecutionProvider,False,False,False,1,False,False,True,False,False,False,False,True,False,False,False,False,True,False,False,False,False,True,True,O3,False,False,False,QOperator,IntegerOps,QUInt8,False,QInt8,True,False,False,"['MatMul', 'Add']",avx2,False,False,inference,src.benchmark.inference.InferenceBenchmark,42,False,False,10,10,1,10,openai/whisper-base,cpu,automatic-speech-recognition,1.8.8.dev0,4.29.2,3.10.12,Linux, Intel(R) Xeon(R) CPU @ 2.20GHz,2,12982,CUDA not available,CUDA not available,0,2.15,0.465,2.32,4.31,"['Gather', 'Transpose', 'EmbedLayerNormalization', 'Attention', 'LSTM', 'ArgMax', 'Gemm', 'MatMul', 'Add', 'Mul', 'Relu', 'Clip', 'LeakyRelu', 'Sigmoid', 'MaxPool', 'GlobalAveragePool', 'Split', 'Pad', 'Reshape', 'Squeeze', 'Unsqueeze', 'Resize', 'AveragePool', 'Concat', 'Softmax', 'Where', 'ConvTranspose', 'InstanceNormalization']",,,,,,23.015873015873023,30.211480362537756
whisper_auto_opt(O1)+auto_qnt(avx2),onnxruntime,1.15.0,,,src.backend.onnxruntime.ORTBackend,CPUExecutionProvider,False,False,False,1,False,False,True,False,False,False,False,True,False,False,False,False,True,False,False,False,False,True,True,O1,False,False,False,QOperator,IntegerOps,QUInt8,False,QInt8,True,False,False,"['MatMul', 'Add']",avx2,False,False,inference,src.benchmark.inference.InferenceBenchmark,42,False,False,10,10,1,10,openai/whisper-base,cpu,automatic-speech-recognition,1.8.8.dev0,4.29.2,3.10.12,Linux, Intel(R) Xeon(R) CPU @ 2.20GHz,2,12982,CUDA not available,CUDA not available,0,2.21,0.452,2.49,4.02,"['Gather', 'Transpose', 'EmbedLayerNormalization', 'Attention', 'LSTM', 'ArgMax', 'Gemm', 'MatMul', 'Add', 'Mul', 'Relu', 'Clip', 'LeakyRelu', 'Sigmoid', 'MaxPool', 'GlobalAveragePool', 'Split', 'Pad', 'Reshape', 'Squeeze', 'Unsqueeze', 'Resize', 'AveragePool', 'Concat', 'Softmax', 'Where', 'ConvTranspose', 'InstanceNormalization']",,,,,,19.576719576719583,21.4501510574018
whisper_baseline,pytorch,2.0.1+cu118,,,src.backend.pytorch.PyTorchBackend,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,inference,src.benchmark.inference.InferenceBenchmark,42,False,False,10,10,1,10,openai/whisper-base,cpu,automatic-speech-recognition,1.8.8.dev0,4.29.2,3.10.12,Linux, Intel(R) Xeon(R) CPU @ 2.20GHz,2,12982,CUDA not available,CUDA not available,0,2.65,0.378,3.02,3.31,,False,False,False,False,False,0.0,0.0
