{
    "Block/struct method, bs= 16x16, v=0, s=b": [
        {
            "annotate": "20",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.72280037842951,
                    "f1": 86.62745564109652
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test2/hp_od-output__squad_test2_es-steps_nte20_ls250_est5000_rn-output__squad_test2_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_abr16_abc16_it0_fw10_r-l1_rfl10_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-100000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 16,
                    "attention_block_rows": 16,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 17.205588718414308,
                    "eval_elapsed_time": 24.35187277989462
                },
                "speedup": 2.243131207946717,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 476160,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 929280,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1405440,
                            "linear_total": 7077888,
                            "nnz": 1415424,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 589568,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1142784,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1732352,
                            "linear_total": 7077888,
                            "nnz": 1742336,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 378624,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 144384,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 523008,
                            "linear_total": 7077888,
                            "nnz": 532992,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 208384,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 431616,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 640000,
                            "linear_total": 7077888,
                            "nnz": 649984,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 628992,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1350144,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1979136,
                            "linear_total": 7077888,
                            "nnz": 1989120,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 913152,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1305600,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2218752,
                            "linear_total": 7077888,
                            "nnz": 2228736,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 850688,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1406976,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2257664,
                            "linear_total": 7077888,
                            "nnz": 2267648,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 764672,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1331712,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2096384,
                            "linear_total": 7077888,
                            "nnz": 2106368,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 763136,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1022976,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1786112,
                            "linear_total": 7077888,
                            "nnz": 1796096,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 781568,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 757248,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1538816,
                            "linear_total": 7077888,
                            "nnz": 1548800,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 596224,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 431616,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1027840,
                            "linear_total": 7077888,
                            "nnz": 1037824,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 394752,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 176640,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 571392,
                            "linear_total": 7077888,
                            "nnz": 581376,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 17776896,
                    "linear_sparsity": 79.0699146412037,
                    "linear_total": 84934656,
                    "nnz": 41735426,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            10,
                            4,
                            6
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 61.67306005721974
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test2",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "output_dir": "output/squad_test2",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test2",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 78.97824030274361,
                    "f1": 86.76378852886562
                }
            },
            "f1": 86.62745564109652,
            "fill_rate": 0.20930085358796302,
            "speedup": 2.243131207946717
        },
        {
            "annotate": "21",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.38505203405866,
                    "f1": 87.07610213911921
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test2/hp_od-output__squad_test2_es-steps_nte20_ls250_est5000_rn-output__squad_test2_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_abr16_abc16_it0_fw10_r-l1_rfl10_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-90000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 16,
                    "attention_block_rows": 16,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 17.41685264968872,
                    "eval_elapsed_time": 24.593657957855612
                },
                "speedup": 2.215922347259501,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 472576,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 964608,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1437184,
                            "linear_total": 7077888,
                            "nnz": 1447168,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 604160,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1150464,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1754624,
                            "linear_total": 7077888,
                            "nnz": 1764608,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 395008,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 145920,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 540928,
                            "linear_total": 7077888,
                            "nnz": 550912,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 217600,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 440832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 658432,
                            "linear_total": 7077888,
                            "nnz": 668416,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 634624,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1380864,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2015488,
                            "linear_total": 7077888,
                            "nnz": 2025472,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 951040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1325568,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2276608,
                            "linear_total": 7077888,
                            "nnz": 2286592,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 861184,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1419264,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2280448,
                            "linear_total": 7077888,
                            "nnz": 2290432,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 779008,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1344000,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2123008,
                            "linear_total": 7077888,
                            "nnz": 2132992,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 799744,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1041408,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1841152,
                            "linear_total": 7077888,
                            "nnz": 1851136,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 790272,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 763392,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1553664,
                            "linear_total": 7077888,
                            "nnz": 1563648,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 610816,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 431616,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1042432,
                            "linear_total": 7077888,
                            "nnz": 1052416,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 405248,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 179712,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 584960,
                            "linear_total": 7077888,
                            "nnz": 594944,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 18108928,
                    "linear_sparsity": 78.6789882330247,
                    "linear_total": 84934656,
                    "nnz": 42067458,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 61.3681447432349
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test2",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "output_dir": "output/squad_test2",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test2",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 79.63103122043519,
                    "f1": 87.27956184118273
                }
            },
            "f1": 87.07610213911921,
            "fill_rate": 0.21321011766975306,
            "speedup": 2.215922347259501
        },
        {
            "annotate": "20",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.74172185430463,
                    "f1": 86.69521763053608
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_abr16_abc16_it0_fw10_r-l1_rfl10_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-100000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 16,
                    "attention_block_rows": 16,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 17.306304389953613,
                    "eval_elapsed_time": 24.480814102105796
                },
                "speedup": 2.230077094204775,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 468992,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 940032,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1409024,
                            "linear_total": 7077888,
                            "nnz": 1419008,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 606208,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1185792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1792000,
                            "linear_total": 7077888,
                            "nnz": 1801984,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 378112,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 147456,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 525568,
                            "linear_total": 7077888,
                            "nnz": 535552,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 207360,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 419328,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 626688,
                            "linear_total": 7077888,
                            "nnz": 636672,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 625664,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1348608,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1974272,
                            "linear_total": 7077888,
                            "nnz": 1984256,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 910592,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1320960,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2231552,
                            "linear_total": 7077888,
                            "nnz": 2241536,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 828672,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1380864,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2209536,
                            "linear_total": 7077888,
                            "nnz": 2219520,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 765440,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1281024,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2046464,
                            "linear_total": 7077888,
                            "nnz": 2056448,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 761088,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1003008,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1764096,
                            "linear_total": 7077888,
                            "nnz": 1774080,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 792832,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 780288,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1573120,
                            "linear_total": 7077888,
                            "nnz": 1583104,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 553728,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 433152,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 986880,
                            "linear_total": 7077888,
                            "nnz": 996864,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 389888,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 182784,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 572672,
                            "linear_total": 7077888,
                            "nnz": 582656,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 17711872,
                    "linear_sparsity": 79.14647231867285,
                    "linear_total": 84934656,
                    "nnz": 41670402,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            10,
                            4,
                            6
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 61.73277361909495
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.69521763053608,
            "fill_rate": 0.20853527681327155,
            "speedup": 2.230077094204775
        },
        {
            "annotate": "20",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.84578997161779,
                    "f1": 86.78133258210022
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_abr16_abc16_it0_fw10_r-l1_rfl10_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 16,
                    "attention_block_rows": 16,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 17.32754041290283,
                    "eval_elapsed_time": 24.51584801170975
                },
                "speedup": 2.2273439903006693,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 465664,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 938496,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1404160,
                            "linear_total": 7077888,
                            "nnz": 1414144,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 584192,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1182720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1766912,
                            "linear_total": 7077888,
                            "nnz": 1776896,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 370432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 145920,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 516352,
                            "linear_total": 7077888,
                            "nnz": 526336,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 200960,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 414720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 615680,
                            "linear_total": 7077888,
                            "nnz": 625664,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 615680,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1345536,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1961216,
                            "linear_total": 7077888,
                            "nnz": 1971200,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 895488,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1314816,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2210304,
                            "linear_total": 7077888,
                            "nnz": 2220288,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 812032,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1377792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2189824,
                            "linear_total": 7077888,
                            "nnz": 2199808,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 755456,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1282560,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2038016,
                            "linear_total": 7077888,
                            "nnz": 2048000,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 739840,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 998400,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1738240,
                            "linear_total": 7077888,
                            "nnz": 1748224,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 797440,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 774144,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1571584,
                            "linear_total": 7077888,
                            "nnz": 1581568,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 513792,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 430080,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 943872,
                            "linear_total": 7077888,
                            "nnz": 953856,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 381184,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 182784,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 563968,
                            "linear_total": 7077888,
                            "nnz": 573952,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 17520128,
                    "linear_sparsity": 79.37222704475309,
                    "linear_total": 84934656,
                    "nnz": 41478658,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            10,
                            4,
                            6
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 61.90885809879785
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.78133258210022,
            "fill_rate": 0.20627772955246915,
            "speedup": 2.2273439903006693
        },
        {
            "annotate": "22",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.73226111636707,
                    "f1": 86.74884583609185
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_abr16_abc16_it0_fw10_r-l1_rfl10_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-75000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 16,
                    "attention_block_rows": 16,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 17.906531421661377,
                    "eval_elapsed_time": 25.082025394309312
                },
                "speedup": 2.155324897745623,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 494336,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1090560,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1584896,
                            "linear_total": 7077888,
                            "nnz": 1594880,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 631552,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1285632,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1917184,
                            "linear_total": 7077888,
                            "nnz": 1927168,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 411392,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 156672,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 568064,
                            "linear_total": 7077888,
                            "nnz": 578048,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 223232,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 460800,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 684032,
                            "linear_total": 7077888,
                            "nnz": 694016,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 648192,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1443840,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2092032,
                            "linear_total": 7077888,
                            "nnz": 2102016,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1047552,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1419264,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2466816,
                            "linear_total": 7077888,
                            "nnz": 2476800,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 942592,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1460736,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2403328,
                            "linear_total": 7077888,
                            "nnz": 2413312,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 837888,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1373184,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2211072,
                            "linear_total": 7077888,
                            "nnz": 2221056,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 841472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1095168,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1936640,
                            "linear_total": 7077888,
                            "nnz": 1946624,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 833536,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 827904,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1661440,
                            "linear_total": 7077888,
                            "nnz": 1671424,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 621824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 462336,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1084160,
                            "linear_total": 7077888,
                            "nnz": 1094144,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 432128,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 188928,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 621056,
                            "linear_total": 7077888,
                            "nnz": 631040,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 19230720,
                    "linear_sparsity": 77.3582175925926,
                    "linear_total": 84934656,
                    "nnz": 43189250,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 60.33796825450584
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 79.09176915799432,
                    "f1": 86.94165524295053
                }
            },
            "f1": 86.74884583609185,
            "fill_rate": 0.22641782407407407,
            "speedup": 2.155324897745623
        },
        {
            "annotate": "21",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.97824030274361,
                    "f1": 86.77789246016766
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_abr16_abc16_it0_fw10_r-l1_rfl10_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-90000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 16,
                    "attention_block_rows": 16,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 17.75180138397217,
                    "eval_elapsed_time": 24.917593302205205
                },
                "speedup": 2.174111357522138,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 480256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 970752,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1451008,
                            "linear_total": 7077888,
                            "nnz": 1460992,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 620288,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1214976,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1835264,
                            "linear_total": 7077888,
                            "nnz": 1845248,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 392704,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 150528,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 543232,
                            "linear_total": 7077888,
                            "nnz": 553216,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 214784,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 434688,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 649472,
                            "linear_total": 7077888,
                            "nnz": 659456,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 625664,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1374720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2000384,
                            "linear_total": 7077888,
                            "nnz": 2010368,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 933376,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1337856,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2271232,
                            "linear_total": 7077888,
                            "nnz": 2281216,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 862464,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1405440,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2267904,
                            "linear_total": 7077888,
                            "nnz": 2277888,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 783616,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1297920,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2081536,
                            "linear_total": 7077888,
                            "nnz": 2091520,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 773376,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1033728,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1807104,
                            "linear_total": 7077888,
                            "nnz": 1817088,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 811008,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 791040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1602048,
                            "linear_total": 7077888,
                            "nnz": 1612032,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 572160,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 437760,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1009920,
                            "linear_total": 7077888,
                            "nnz": 1019904,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 405504,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 187392,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 592896,
                            "linear_total": 7077888,
                            "nnz": 602880,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 18112000,
                    "linear_sparsity": 78.67537133487654,
                    "linear_total": 84934656,
                    "nnz": 42070530,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 61.365323630075444
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 79.11069063386944,
                    "f1": 86.88040012719469
                }
            },
            "f1": 86.77789246016766,
            "fill_rate": 0.21324628665123457,
            "speedup": 2.174111357522138
        },
        {
            "annotate": "11",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 76.33869441816462,
                    "f1": 84.90005817955239
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_abr16_abc16_it0_fw10_r-l1_rfl30_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 16,
                    "attention_block_rows": 16,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 30
                },
                "speed": {
                    "cuda_eval_elapsed_time": 14.06190474319458,
                    "eval_elapsed_time": 21.232633945997804
                },
                "speedup": 2.744606346736156,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 295680,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 405504,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 701184,
                            "linear_total": 7077888,
                            "nnz": 711168,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 380672,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 662016,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1042688,
                            "linear_total": 7077888,
                            "nnz": 1052672,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 202240,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 113664,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 315904,
                            "linear_total": 7077888,
                            "nnz": 325888,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 129536,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 178176,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 307712,
                            "linear_total": 7077888,
                            "nnz": 317696,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 328960,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 758784,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1087744,
                            "linear_total": 7077888,
                            "nnz": 1097728,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 612608,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 728064,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1340672,
                            "linear_total": 7077888,
                            "nnz": 1350656,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 331776,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 811008,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1142784,
                            "linear_total": 7077888,
                            "nnz": 1152768,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 411136,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 754176,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1165312,
                            "linear_total": 7077888,
                            "nnz": 1175296,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 319744,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 588288,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 908032,
                            "linear_total": 7077888,
                            "nnz": 918016,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 457472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 499200,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 956672,
                            "linear_total": 7077888,
                            "nnz": 966656,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 246784,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 310272,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 557056,
                            "linear_total": 7077888,
                            "nnz": 567040,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 252672,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 107520,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 360192,
                            "linear_total": 7077888,
                            "nnz": 370176,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 9885952,
                    "linear_sparsity": 88.36052035108025,
                    "linear_total": 84934656,
                    "nnz": 33844482,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            9,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            6,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 68.91955939281638
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 76.71712393566698,
                    "f1": 85.09538457703842
                }
            },
            "f1": 84.90005817955239,
            "fill_rate": 0.11639479648919748,
            "speedup": 2.744606346736156
        }
    ],
    "Block/struct method, bs= 16x16, v=1, s=b": [
        {
            "annotate": "35",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.93661305581836,
                    "f1": 88.35425478567389
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v5-a16-l5--2021-01-17--14-55-26/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 16,
                    "attention_block_rows": 16,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 5.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 23.427229469299316,
                    "eval_elapsed_time": 30.796412555966526
                },
                "speedup": 1.6474160145973682,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 878336,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1947648,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2825984,
                            "linear_total": 7077888,
                            "nnz": 2833092,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 852736,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1970688,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2823424,
                            "linear_total": 7077888,
                            "nnz": 2830419,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 583168,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 228864,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 812032,
                            "linear_total": 7077888,
                            "nnz": 817797,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 385792,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 824832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1210624,
                            "linear_total": 7077888,
                            "nnz": 1216601,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1168384,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2198016,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3366400,
                            "linear_total": 7077888,
                            "nnz": 3373991,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1360384,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2168832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3529216,
                            "linear_total": 7077888,
                            "nnz": 3536900,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1525248,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2168832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3694080,
                            "linear_total": 7077888,
                            "nnz": 3702100,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1519360,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2042880,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3562240,
                            "linear_total": 7077888,
                            "nnz": 3570098,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1345792,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1598976,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2944768,
                            "linear_total": 7077888,
                            "nnz": 2952225,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 1175296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1274880,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2450176,
                            "linear_total": 7077888,
                            "nnz": 2457134,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 1126912,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 701952,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1828864,
                            "linear_total": 7077888,
                            "nnz": 1835577,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 702464,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 321024,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1023488,
                            "linear_total": 7077888,
                            "nnz": 1029361,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 30071296,
                    "linear_sparsity": 64.5947868441358,
                    "linear_total": 84934656,
                    "nnz": 53994017,
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            5,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7
                        ],
                        "11": [
                            0,
                            5,
                            6,
                            7,
                            8,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            3,
                            7
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 50.41561461889819
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.35425478567389,
            "fill_rate": 0.354052131558642,
            "speedup": 1.6474160145973682
        },
        {
            "annotate": "35",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 81.11636707663197,
                    "f1": 88.26635621180897
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v5-a16-l5--2021-01-17--14-55-26/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 16,
                    "attention_block_rows": 16,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 5.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 23.067204750061034,
                    "eval_elapsed_time": 30.552880198229104
                },
                "speedup": 1.6731282972319816,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 878336,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1947648,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2825984,
                            "linear_total": 7077888,
                            "nnz": 2833092,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 852736,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1970688,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2823424,
                            "linear_total": 7077888,
                            "nnz": 2830419,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 583168,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 228864,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 812032,
                            "linear_total": 7077888,
                            "nnz": 817797,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 385792,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 824832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1210624,
                            "linear_total": 7077888,
                            "nnz": 1216601,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1168384,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2198016,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3366400,
                            "linear_total": 7077888,
                            "nnz": 3373991,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1360384,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2168832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3529216,
                            "linear_total": 7077888,
                            "nnz": 3536900,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1525248,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2168832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3694080,
                            "linear_total": 7077888,
                            "nnz": 3702100,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1519360,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2042880,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3562240,
                            "linear_total": 7077888,
                            "nnz": 3570098,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1345792,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1598976,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2944768,
                            "linear_total": 7077888,
                            "nnz": 2952225,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 1175296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1274880,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2450176,
                            "linear_total": 7077888,
                            "nnz": 2457134,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 1126912,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 701952,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1828864,
                            "linear_total": 7077888,
                            "nnz": 1835577,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 702464,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 321024,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1023488,
                            "linear_total": 7077888,
                            "nnz": 1029361,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 30071296,
                    "linear_sparsity": 64.5947868441358,
                    "linear_total": 84934656,
                    "nnz": 53994017,
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            5,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7
                        ],
                        "11": [
                            0,
                            5,
                            6,
                            7,
                            8,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            3,
                            7
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 50.41561461889819
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.26635621180897,
            "fill_rate": 0.354052131558642,
            "speedup": 1.6731282972319816
        }
    ],
    "Block/struct method, bs= 32x32, v=0, s=b": [
        {
            "annotate": "21",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.25260170293284,
                    "f1": 86.93528973939952
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test/hp_es-steps_nte20_ls250_est5000_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_abr32_abc32_it0_fw10_r-l1_rfl10_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-105000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 16.44181579208374,
                    "eval_elapsed_time": 23.62707085069269
                },
                "speedup": 2.3473315534865185,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 542720,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 973824,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1516544,
                            "linear_total": 7077888,
                            "nnz": 1526528,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 564992,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1193472,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1758464,
                            "linear_total": 7077888,
                            "nnz": 1768448,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 354560,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 167424,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 521984,
                            "linear_total": 7077888,
                            "nnz": 531968,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 231680,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 423936,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 655616,
                            "linear_total": 7077888,
                            "nnz": 665600,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 646144,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1383936,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2030080,
                            "linear_total": 7077888,
                            "nnz": 2040064,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 969472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1359360,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2328832,
                            "linear_total": 7077888,
                            "nnz": 2338816,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 857856,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1425408,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2283264,
                            "linear_total": 7077888,
                            "nnz": 2293248,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 702976,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1396224,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2099200,
                            "linear_total": 7077888,
                            "nnz": 2109184,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 774656,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1072128,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1846784,
                            "linear_total": 7077888,
                            "nnz": 1856768,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 806400,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 783360,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1589760,
                            "linear_total": 7077888,
                            "nnz": 1599744,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 520448,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 446976,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 967424,
                            "linear_total": 7077888,
                            "nnz": 977408,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 435968,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 181248,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 617216,
                            "linear_total": 7077888,
                            "nnz": 627200,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 18215168,
                    "linear_sparsity": 78.55390383873457,
                    "linear_total": 84934656,
                    "nnz": 42173698,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 61.27058124647028
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "output_dir": "output/squad_test",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 79.46073793755913,
                    "f1": 87.08591835424342
                }
            },
            "f1": 86.93528973939952,
            "fill_rate": 0.21446096161265427,
            "speedup": 2.3473315534865185
        },
        {
            "annotate": "21",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.2620624408704,
                    "f1": 86.97825692623259
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test/hp_es-steps_nte20_ls250_est5000_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_abr32_abc32_it0_fw10_r-l1_rfl10_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 16.405798454284668,
                    "eval_elapsed_time": 23.622337056789547
                },
                "speedup": 2.3524848920286154,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 519424,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 973824,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1493248,
                            "linear_total": 7077888,
                            "nnz": 1503232,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 565504,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1191936,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1757440,
                            "linear_total": 7077888,
                            "nnz": 1767424,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 346368,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 167424,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 513792,
                            "linear_total": 7077888,
                            "nnz": 523776,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 220160,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 423936,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 644096,
                            "linear_total": 7077888,
                            "nnz": 654080,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 646400,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1382400,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2028800,
                            "linear_total": 7077888,
                            "nnz": 2038784,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 937728,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1359360,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2297088,
                            "linear_total": 7077888,
                            "nnz": 2307072,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 846592,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1423872,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2270464,
                            "linear_total": 7077888,
                            "nnz": 2280448,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 688640,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1393152,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2081792,
                            "linear_total": 7077888,
                            "nnz": 2091776,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 744704,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1070592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1815296,
                            "linear_total": 7077888,
                            "nnz": 1825280,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 831488,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 781824,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1613312,
                            "linear_total": 7077888,
                            "nnz": 1623296,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 522496,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 446976,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 969472,
                            "linear_total": 7077888,
                            "nnz": 979456,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 413696,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 181248,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 594944,
                            "linear_total": 7077888,
                            "nnz": 604928,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 18079744,
                    "linear_sparsity": 78.7133487654321,
                    "linear_total": 84934656,
                    "nnz": 42038274,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 61.39494531824976
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "output_dir": "output/squad_test",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.97825692623259,
            "fill_rate": 0.212866512345679,
            "speedup": 2.3524848920286154
        },
        {
            "annotate": "29",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.6244087038789,
                    "f1": 88.07723643002453
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test/hp_es-steps_nte20_ls250_est5000_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_abr32_abc32_it0_fw10_r-l1_rfl5_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 5
                },
                "speed": {
                    "cuda_eval_elapsed_time": 19.890604362487792,
                    "eval_elapsed_time": 27.08285549096763
                },
                "speedup": 1.9403328477116193,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 721408,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1492992,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2214400,
                            "linear_total": 7077888,
                            "nnz": 2224384,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 635136,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1755648,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2390784,
                            "linear_total": 7077888,
                            "nnz": 2400768,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 484608,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 198144,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 682752,
                            "linear_total": 7077888,
                            "nnz": 692736,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 313600,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 666624,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 980224,
                            "linear_total": 7077888,
                            "nnz": 990208,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 972032,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1878528,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2850560,
                            "linear_total": 7077888,
                            "nnz": 2860544,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1256448,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1932288,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3188736,
                            "linear_total": 7077888,
                            "nnz": 3198720,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1260544,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1889280,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3149824,
                            "linear_total": 7077888,
                            "nnz": 3159808,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1121280,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1784832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2906112,
                            "linear_total": 7077888,
                            "nnz": 2916096,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1061888,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1393152,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2455040,
                            "linear_total": 7077888,
                            "nnz": 2465024,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 988160,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1027584,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2015744,
                            "linear_total": 7077888,
                            "nnz": 2025728,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 903424,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 646656,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1550080,
                            "linear_total": 7077888,
                            "nnz": 1560064,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 636416,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 250368,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 886784,
                            "linear_total": 7077888,
                            "nnz": 896768,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 25271040,
                    "linear_sparsity": 70.2464916087963,
                    "linear_total": 84934656,
                    "nnz": 49229570,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            1,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 54.79095450471988
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "output_dir": "output/squad_test",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.07723643002453,
            "fill_rate": 0.297535083912037,
            "speedup": 1.9403328477116193
        },
        {
            "annotate": "31",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.99053926206244,
                    "f1": 87.56439208763325
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test2/hp_od-output__squad_test2_es-steps_nte12_ls250_est5000_rn-output__squad_test2_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_abr32_abc32_it0_r-l1_rfl10_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-65000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 2,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 19.37784966278076,
                    "eval_elapsed_time": 26.613120706751943
                },
                "speedup": 1.9916757368336773,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 684800,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2007552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2692352,
                            "linear_total": 7077888,
                            "nnz": 2702336,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 646656,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2019840,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2666496,
                            "linear_total": 7077888,
                            "nnz": 2676480,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 432128,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 274944,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 707072,
                            "linear_total": 7077888,
                            "nnz": 717056,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 277760,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 794112,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1071872,
                            "linear_total": 7077888,
                            "nnz": 1081856,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 691712,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2239488,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2931200,
                            "linear_total": 7077888,
                            "nnz": 2941184,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1149184,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2211840,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3361024,
                            "linear_total": 7077888,
                            "nnz": 3371008,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1007872,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2158080,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3165952,
                            "linear_total": 7077888,
                            "nnz": 3175936,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 997376,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2073600,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3070976,
                            "linear_total": 7077888,
                            "nnz": 3080960,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 911872,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1732608,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2644480,
                            "linear_total": 7077888,
                            "nnz": 2654464,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 944640,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1304064,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2248704,
                            "linear_total": 7077888,
                            "nnz": 2258688,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 763136,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 751104,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1514240,
                            "linear_total": 7077888,
                            "nnz": 1524224,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 526080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 313344,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 839424,
                            "linear_total": 7077888,
                            "nnz": 849408,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 26913792,
                    "linear_sparsity": 68.31235532407408,
                    "linear_total": 84934656,
                    "nnz": 50872322,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 53.282364242699266
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test2",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 12,
                    "output_dir": "output/squad_test2",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test2",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.56439208763325,
            "fill_rate": 0.3168764467592592,
            "speedup": 1.9916757368336773
        },
        {
            "annotate": "26",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.39451277199622,
                    "f1": 86.84346997900737
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test2/hp_od-output__squad_test2_es-steps_nte12_ls250_est5000_rn-output__squad_test2_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_abr32_abc32_it0_r-l1_rfl15_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-65000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 2,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 15
                },
                "speed": {
                    "cuda_eval_elapsed_time": 17.153405197143556,
                    "eval_elapsed_time": 24.343701715115458
                },
                "speedup": 2.249955187428206,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 551680,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1539072,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2090752,
                            "linear_total": 7077888,
                            "nnz": 2100736,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 596736,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1681920,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2278656,
                            "linear_total": 7077888,
                            "nnz": 2288640,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 361728,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 236544,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 598272,
                            "linear_total": 7077888,
                            "nnz": 608256,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 238336,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 620544,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 858880,
                            "linear_total": 7077888,
                            "nnz": 868864,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 567808,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1850880,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2418688,
                            "linear_total": 7077888,
                            "nnz": 2428672,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1002752,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1841664,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2844416,
                            "linear_total": 7077888,
                            "nnz": 2854400,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 878592,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1812480,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2691072,
                            "linear_total": 7077888,
                            "nnz": 2701056,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 721152,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1754112,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2475264,
                            "linear_total": 7077888,
                            "nnz": 2485248,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 805376,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1423872,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2229248,
                            "linear_total": 7077888,
                            "nnz": 2239232,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 892672,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1073664,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1966336,
                            "linear_total": 7077888,
                            "nnz": 1976320,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 460800,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 620544,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1081344,
                            "linear_total": 7077888,
                            "nnz": 1091328,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 454144,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 247296,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 701440,
                            "linear_total": 7077888,
                            "nnz": 711424,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 22234368,
                    "linear_sparsity": 73.82179542824075,
                    "linear_total": 84934656,
                    "nnz": 46192898,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 57.57962486284496
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test2",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 12,
                    "output_dir": "output/squad_test2",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test2",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 79.7918637653737,
                    "f1": 87.14951283583915
                }
            },
            "f1": 86.84346997900737,
            "fill_rate": 0.26178204571759256,
            "speedup": 2.249955187428206
        },
        {
            "annotate": "17",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.7038789025544,
                    "f1": 86.58426699451658
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test2/hp_od-output__squad_test2_es-steps_nte20_ls250_est5000_rn-output__squad_test2_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_abr32_abc32_it0_fw10_r-l1_rfl15_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 15
                },
                "speed": {
                    "cuda_eval_elapsed_time": 15.051653835296632,
                    "eval_elapsed_time": 22.226274209097028
                },
                "speedup": 2.56412972472606,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 459776,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 749568,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1209344,
                            "linear_total": 7077888,
                            "nnz": 1219328,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 488192,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1006080,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1494272,
                            "linear_total": 7077888,
                            "nnz": 1504256,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 311040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 148992,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 460032,
                            "linear_total": 7077888,
                            "nnz": 470016,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 207360,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 311808,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 519168,
                            "linear_total": 7077888,
                            "nnz": 529152,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 550144,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1085952,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1636096,
                            "linear_total": 7077888,
                            "nnz": 1646080,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 868352,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1101312,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1969664,
                            "linear_total": 7077888,
                            "nnz": 1979648,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 548864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1198080,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1746944,
                            "linear_total": 7077888,
                            "nnz": 1756928,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 653312,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1128960,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1782272,
                            "linear_total": 7077888,
                            "nnz": 1792256,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 593920,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 867840,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1461760,
                            "linear_total": 7077888,
                            "nnz": 1471744,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 721920,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 669696,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1391616,
                            "linear_total": 7077888,
                            "nnz": 1401600,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 367616,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 387072,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 754688,
                            "linear_total": 7077888,
                            "nnz": 764672,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 373760,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 158208,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 531968,
                            "linear_total": 7077888,
                            "nnz": 541952,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 14957824,
                    "linear_sparsity": 82.38902150848766,
                    "linear_total": 84934656,
                    "nnz": 38916354,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 64.26190156654981
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test2",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "output_dir": "output/squad_test2",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test2",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.58426699451658,
            "fill_rate": 0.1761097849151234,
            "speedup": 2.56412972472606
        },
        {
            "annotate": "98",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.90823084200568,
                    "f1": 88.13888839423888
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_abr32_abc32_it0_fw10_r-l1_rfl14.99999_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-10000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": true,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 14.99999
                },
                "speed": {
                    "cuda_eval_elapsed_time": 40.403957000732426,
                    "eval_elapsed_time": 47.70582241564989
                },
                "speedup": 0.9552131986644643,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 2151936,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4713984,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 6865920,
                            "linear_total": 7077888,
                            "nnz": 6875904,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 2299648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4709376,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7009024,
                            "linear_total": 7077888,
                            "nnz": 7019008,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 2285568,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4638720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 6924288,
                            "linear_total": 7077888,
                            "nnz": 6934272,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 2312448,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4687872,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7000320,
                            "linear_total": 7077888,
                            "nnz": 7010304,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 2330112,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4707840,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7037952,
                            "linear_total": 7077888,
                            "nnz": 7047936,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 2330112,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4710912,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7041024,
                            "linear_total": 7077888,
                            "nnz": 7051008,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 2324992,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4704768,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7029760,
                            "linear_total": 7077888,
                            "nnz": 7039744,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 2337280,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4706304,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7043584,
                            "linear_total": 7077888,
                            "nnz": 7053568,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 2321664,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4684800,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7006464,
                            "linear_total": 7077888,
                            "nnz": 7016448,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 2342400,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4683264,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7025664,
                            "linear_total": 7077888,
                            "nnz": 7035648,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 2296576,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4654080,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 6950656,
                            "linear_total": 7077888,
                            "nnz": 6960640,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 2259200,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4646400,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 6905600,
                            "linear_total": 7077888,
                            "nnz": 6915584,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 83840256,
                    "linear_sparsity": 1.288519965277779,
                    "linear_total": 84934656,
                    "nnz": 107798786,
                    "pruned_heads": {
                        "0": [],
                        "1": [],
                        "10": [],
                        "11": [],
                        "2": [],
                        "3": [],
                        "4": [],
                        "5": [],
                        "6": [],
                        "7": [],
                        "8": [],
                        "9": []
                    },
                    "total": 108893186,
                    "total_sparsity": 1.005021563057218
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.13888839423888,
            "fill_rate": 0.9871148003472222,
            "speedup": 0.9552131986644643
        },
        {
            "annotate": "94",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.21192052980132,
                    "f1": 86.2154189083501
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_abr32_abc32_it0_fw10_r-l1_rfl14.99999_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-15000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": true,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 14.99999
                },
                "speed": {
                    "cuda_eval_elapsed_time": 39.830447120666506,
                    "eval_elapsed_time": 47.13309640903026
                },
                "speedup": 0.968967104196677,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1914624,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4678656,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 6593280,
                            "linear_total": 7077888,
                            "nnz": 6603264,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 2103296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4669440,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 6772736,
                            "linear_total": 7077888,
                            "nnz": 6782720,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 2053632,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4353024,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 6406656,
                            "linear_total": 7077888,
                            "nnz": 6416640,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 2100480,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4538880,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 6639360,
                            "linear_total": 7077888,
                            "nnz": 6649344,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 2239232,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4646400,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 6885632,
                            "linear_total": 7077888,
                            "nnz": 6895616,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 2219520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4657152,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 6876672,
                            "linear_total": 7077888,
                            "nnz": 6886656,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 2216448,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4657152,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 6873600,
                            "linear_total": 7077888,
                            "nnz": 6883584,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 2226176,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4615680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 6841856,
                            "linear_total": 7077888,
                            "nnz": 6851840,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 2190848,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4595712,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 6786560,
                            "linear_total": 7077888,
                            "nnz": 6796544,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 2261760,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4549632,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 6811392,
                            "linear_total": 7077888,
                            "nnz": 6821376,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 2178048,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4431360,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 6609408,
                            "linear_total": 7077888,
                            "nnz": 6619392,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 2049792,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4349952,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 6399744,
                            "linear_total": 7077888,
                            "nnz": 6409728,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 80496896,
                    "linear_sparsity": 5.224910783179015,
                    "linear_total": 84934656,
                    "nnz": 104455426,
                    "pruned_heads": {
                        "0": [],
                        "1": [],
                        "10": [],
                        "11": [],
                        "2": [],
                        "3": [],
                        "4": [],
                        "5": [],
                        "6": [],
                        "7": [],
                        "8": [],
                        "9": []
                    },
                    "total": 108893186,
                    "total_sparsity": 4.075333051601593
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.2154189083501,
            "fill_rate": 0.9477508921682098,
            "speedup": 0.968967104196677
        },
        {
            "annotate": "99",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.7038789025544,
                    "f1": 86.6699349353281
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_abr32_abc32_it0_fw10_r-l1_rfl14.9999_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-5000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": true,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 14.9999
                },
                "speed": {
                    "cuda_eval_elapsed_time": 39.58176746368408,
                    "eval_elapsed_time": 46.91258597606793
                },
                "speedup": 0.975054816356574,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 2354176,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7072768,
                            "linear_total": 7077888,
                            "nnz": 7082752,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4715520,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7074816,
                            "linear_total": 7077888,
                            "nnz": 7084800,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4717056,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7076352,
                            "linear_total": 7077888,
                            "nnz": 7086336,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 2358272,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7076864,
                            "linear_total": 7077888,
                            "nnz": 7086848,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 2358272,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7076864,
                            "linear_total": 7077888,
                            "nnz": 7086848,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 84922880,
                    "linear_sparsity": 0.013864776234573384,
                    "linear_total": 84934656,
                    "nnz": 108881410,
                    "pruned_heads": {
                        "0": [],
                        "1": [],
                        "10": [],
                        "11": [],
                        "2": [],
                        "3": [],
                        "4": [],
                        "5": [],
                        "6": [],
                        "7": [],
                        "8": [],
                        "9": []
                    },
                    "total": 108893186,
                    "total_sparsity": 0.010814267111258768
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.6699349353281,
            "fill_rate": 0.9998613522376543,
            "speedup": 0.975054816356574
        },
        {
            "annotate": "15",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.94701986754967,
                    "f1": 86.06827252573265
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_abr32_abc32_it0_fw10_r-l1_rfl20_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20
                },
                "speed": {
                    "cuda_eval_elapsed_time": 14.216132064819336,
                    "eval_elapsed_time": 21.342612544074655
                },
                "speedup": 2.7148307872632,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 439296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 605184,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1044480,
                            "linear_total": 7077888,
                            "nnz": 1054464,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 367616,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 809472,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1177088,
                            "linear_total": 7077888,
                            "nnz": 1187072,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 276224,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 135168,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 411392,
                            "linear_total": 7077888,
                            "nnz": 421376,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 178176,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 251904,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 430080,
                            "linear_total": 7077888,
                            "nnz": 440064,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 492032,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 958464,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1450496,
                            "linear_total": 7077888,
                            "nnz": 1460480,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 733696,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 918528,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1652224,
                            "linear_total": 7077888,
                            "nnz": 1662208,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 461056,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1050624,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1511680,
                            "linear_total": 7077888,
                            "nnz": 1521664,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 580096,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 953856,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1533952,
                            "linear_total": 7077888,
                            "nnz": 1543936,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 462592,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 764928,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1227520,
                            "linear_total": 7077888,
                            "nnz": 1237504,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 624384,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 571392,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1195776,
                            "linear_total": 7077888,
                            "nnz": 1205760,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 351744,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 348672,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 700416,
                            "linear_total": 7077888,
                            "nnz": 710400,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 339968,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 139776,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 479744,
                            "linear_total": 7077888,
                            "nnz": 489728,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 12814848,
                    "linear_sparsity": 84.912109375,
                    "linear_total": 84934656,
                    "nnz": 36773378,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            3,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            9,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            6,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 66.22986308803564
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.06827252573265,
            "fill_rate": 0.15087890625,
            "speedup": 2.7148307872632
        },
        {
            "annotate": "13",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.06717123935667,
                    "f1": 85.28341140334766
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_abr32_abc32_it0_fw10_r-l1_rfl25_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 25
                },
                "speed": {
                    "cuda_eval_elapsed_time": 13.584790561676026,
                    "eval_elapsed_time": 20.705443068873137
                },
                "speedup": 2.8410002222816386,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 384768,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 502272,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 887040,
                            "linear_total": 7077888,
                            "nnz": 897024,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 355840,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 701952,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1057792,
                            "linear_total": 7077888,
                            "nnz": 1067776,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 256512,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 115200,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 371712,
                            "linear_total": 7077888,
                            "nnz": 381696,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 150016,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 221184,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 371200,
                            "linear_total": 7077888,
                            "nnz": 381184,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 413440,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 872448,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1285888,
                            "linear_total": 7077888,
                            "nnz": 1295872,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 672256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 824832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1497088,
                            "linear_total": 7077888,
                            "nnz": 1507072,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 418560,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 932352,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1350912,
                            "linear_total": 7077888,
                            "nnz": 1360896,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 523264,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 872448,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1395712,
                            "linear_total": 7077888,
                            "nnz": 1405696,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 498944,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 655872,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1154816,
                            "linear_total": 7077888,
                            "nnz": 1164800,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 497664,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 562176,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1059840,
                            "linear_total": 7077888,
                            "nnz": 1069824,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 297216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 311808,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 609024,
                            "linear_total": 7077888,
                            "nnz": 619008,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 316416,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 119808,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 436224,
                            "linear_total": 7077888,
                            "nnz": 446208,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 11477248,
                    "linear_sparsity": 86.4869671103395,
                    "linear_total": 84934656,
                    "nnz": 35435778,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            10,
                            11
                        ],
                        "3": [
                            2,
                            3,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            9,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            2,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 67.45822277621669
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 85.28341140334766,
            "fill_rate": 0.13513032889660492,
            "speedup": 2.8410002222816386
        }
    ],
    "Block/struct method, bs= 32x32, v=1, s=b": [
        {
            "annotate": "26",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.80132450331126,
                    "f1": 87.48291010744668
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl10_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 18.270113506317138,
                    "eval_elapsed_time": 25.450434973929077
                },
                "speedup": 2.1124331270315624,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 627712,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1281024,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1908736,
                            "linear_total": 7077888,
                            "nnz": 1914914,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 596992,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1548288,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2145280,
                            "linear_total": 7077888,
                            "nnz": 2151632,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 451584,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 182784,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 634368,
                            "linear_total": 7077888,
                            "nnz": 639895,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 268288,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 559104,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 827392,
                            "linear_total": 7077888,
                            "nnz": 832812,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 789504,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1709568,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2499072,
                            "linear_total": 7077888,
                            "nnz": 2505785,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1180672,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1740288,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2920960,
                            "linear_total": 7077888,
                            "nnz": 2928205,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1204224,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1701888,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2906112,
                            "linear_total": 7077888,
                            "nnz": 2913428,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 916480,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1600512,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2516992,
                            "linear_total": 7077888,
                            "nnz": 2523794,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 909312,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1242624,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2151936,
                            "linear_total": 7077888,
                            "nnz": 2158569,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 917504,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 972288,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1889792,
                            "linear_total": 7077888,
                            "nnz": 1896217,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 856064,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 542208,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1398272,
                            "linear_total": 7077888,
                            "nnz": 1404481,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 611328,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 247296,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 858624,
                            "linear_total": 7077888,
                            "nnz": 864321,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 22657536,
                    "linear_sparsity": 73.32356770833333,
                    "linear_total": 84934656,
                    "nnz": 46572775,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 57.23077199706509
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.48291010744668,
            "fill_rate": 0.26676432291666674,
            "speedup": 2.1124331270315624
        },
        {
            "annotate": "27",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.27436140018922,
                    "f1": 87.70461789964966
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl10_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-90000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 18.375184078216552,
                    "eval_elapsed_time": 25.600778602063656
                },
                "speedup": 2.1003540884863323,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 645120,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1339392,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1984512,
                            "linear_total": 7077888,
                            "nnz": 1990760,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 592896,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1571328,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2164224,
                            "linear_total": 7077888,
                            "nnz": 2170591,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 480256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 187392,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 667648,
                            "linear_total": 7077888,
                            "nnz": 673242,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 294912,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 574464,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 869376,
                            "linear_total": 7077888,
                            "nnz": 874838,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 880640,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1744896,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2625536,
                            "linear_total": 7077888,
                            "nnz": 2632432,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1230848,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1761792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2992640,
                            "linear_total": 7077888,
                            "nnz": 2999931,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1214464,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1726464,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2940928,
                            "linear_total": 7077888,
                            "nnz": 2948260,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 906240,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1629696,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2535936,
                            "linear_total": 7077888,
                            "nnz": 2542789,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 943104,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1270272,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2213376,
                            "linear_total": 7077888,
                            "nnz": 2220027,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 935936,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 987648,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1923584,
                            "linear_total": 7077888,
                            "nnz": 1930019,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 872448,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 546816,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1419264,
                            "linear_total": 7077888,
                            "nnz": 1425508,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 634880,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 248832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 883712,
                            "linear_total": 7077888,
                            "nnz": 889410,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 23220736,
                    "linear_sparsity": 72.66046971450618,
                    "linear_total": 84934656,
                    "nnz": 47136529,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            11,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 56.713059162397904
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.70461789964966,
            "fill_rate": 0.2733953028549382,
            "speedup": 2.1003540884863323
        },
        {
            "annotate": "18",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.20529801324503,
                    "f1": 87.11181141207972
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl10_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 17.401466148376464,
                    "eval_elapsed_time": 24.569451212882996
                },
                "speedup": 2.2178816817094407,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 838656,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 287232,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1125888,
                            "linear_total": 7077888,
                            "nnz": 1131739,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 692224,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 496128,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1188352,
                            "linear_total": 7077888,
                            "nnz": 1194115,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 489472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 84480,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 573952,
                            "linear_total": 7077888,
                            "nnz": 579479,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 293888,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 155136,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 449024,
                            "linear_total": 7077888,
                            "nnz": 454245,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1089536,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 605184,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1694720,
                            "linear_total": 7077888,
                            "nnz": 1701098,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1291264,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 671232,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1962496,
                            "linear_total": 7077888,
                            "nnz": 1969077,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1384448,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 728064,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2112512,
                            "linear_total": 7077888,
                            "nnz": 2119386,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1121280,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 662016,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1783296,
                            "linear_total": 7077888,
                            "nnz": 1789807,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1127424,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 505344,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1632768,
                            "linear_total": 7077888,
                            "nnz": 1639241,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 942080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 391680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1333760,
                            "linear_total": 7077888,
                            "nnz": 1339807,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 982016,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 222720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1204736,
                            "linear_total": 7077888,
                            "nnz": 1210897,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 645120,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 84480,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 729600,
                            "linear_total": 7077888,
                            "nnz": 735223,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 15791104,
                    "linear_sparsity": 81.40793788580247,
                    "linear_total": 84934656,
                    "nnz": 39702836,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7
                        ],
                        "6": [
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 63.53965068117302
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.11181141207972,
            "fill_rate": 0.18592062114197527,
            "speedup": 2.2178816817094407
        },
        {
            "annotate": "19",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.80794701986756,
                    "f1": 86.74156854566804
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl20_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20
                },
                "speed": {
                    "cuda_eval_elapsed_time": 15.618790004730226,
                    "eval_elapsed_time": 22.811819266993552
                },
                "speedup": 2.471023235070233,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 518144,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 826368,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1344512,
                            "linear_total": 7077888,
                            "nnz": 1350394,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 516096,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1090560,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1606656,
                            "linear_total": 7077888,
                            "nnz": 1612614,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 324608,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 147456,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 472064,
                            "linear_total": 7077888,
                            "nnz": 477376,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 209920,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 345600,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 555520,
                            "linear_total": 7077888,
                            "nnz": 560737,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 637952,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1204224,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1842176,
                            "linear_total": 7077888,
                            "nnz": 1848464,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 913408,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1184256,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2097664,
                            "linear_total": 7077888,
                            "nnz": 2104259,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 790528,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1265664,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2056192,
                            "linear_total": 7077888,
                            "nnz": 2062744,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 664576,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1201152,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1865728,
                            "linear_total": 7077888,
                            "nnz": 1871982,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 629760,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 935424,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1565184,
                            "linear_total": 7077888,
                            "nnz": 1571297,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 787456,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 698880,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1486336,
                            "linear_total": 7077888,
                            "nnz": 1492487,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 415744,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 428544,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 844288,
                            "linear_total": 7077888,
                            "nnz": 849783,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 423936,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 168960,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 592896,
                            "linear_total": 7077888,
                            "nnz": 598254,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 16329216,
                    "linear_sparsity": 80.7743778935185,
                    "linear_total": 84934656,
                    "nnz": 40239113,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 63.04717083032174
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.74156854566804,
            "fill_rate": 0.192256221064815,
            "speedup": 2.471023235070233
        },
        {
            "annotate": "19",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.68495742667928,
                    "f1": 86.66781681977909
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl20_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-90000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20
                },
                "speed": {
                    "cuda_eval_elapsed_time": 15.796802043914795,
                    "eval_elapsed_time": 22.965831307694316
                },
                "speedup": 2.4431776063331965,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 546816,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 847872,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1394688,
                            "linear_total": 7077888,
                            "nnz": 1400584,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 539648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1101312,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1640960,
                            "linear_total": 7077888,
                            "nnz": 1646989,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 354304,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 147456,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 501760,
                            "linear_total": 7077888,
                            "nnz": 507136,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 226304,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 365568,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 591872,
                            "linear_total": 7077888,
                            "nnz": 597102,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 657408,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1221120,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1878528,
                            "linear_total": 7077888,
                            "nnz": 1884827,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 931840,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1211904,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2143744,
                            "linear_total": 7077888,
                            "nnz": 2150389,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 864256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1279488,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2143744,
                            "linear_total": 7077888,
                            "nnz": 2150337,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 686080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1216512,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1902592,
                            "linear_total": 7077888,
                            "nnz": 1908856,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 649216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 952320,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1601536,
                            "linear_total": 7077888,
                            "nnz": 1607660,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 791552,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 715776,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1507328,
                            "linear_total": 7077888,
                            "nnz": 1513490,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 474112,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 434688,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 908800,
                            "linear_total": 7077888,
                            "nnz": 914459,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 435200,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 172032,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 607232,
                            "linear_total": 7077888,
                            "nnz": 612624,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 16822784,
                    "linear_sparsity": 80.19326292438271,
                    "linear_total": 84934656,
                    "nnz": 40733175,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 62.593458327135366
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 78.85525070955535,
                    "f1": 86.76897969849135
                }
            },
            "f1": 86.66781681977909,
            "fill_rate": 0.19806737075617287,
            "speedup": 2.4431776063331965
        },
        {
            "annotate": "13",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.18070009460737,
                    "f1": 85.6109462422114
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl40_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 40
                },
                "speed": {
                    "cuda_eval_elapsed_time": 13.485522186279297,
                    "eval_elapsed_time": 20.651509277056903
                },
                "speedup": 2.86191312967017,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 424960,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 482304,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 907264,
                            "linear_total": 7077888,
                            "nnz": 912794,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 367616,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 706560,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1074176,
                            "linear_total": 7077888,
                            "nnz": 1079692,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 256000,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 121344,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 377344,
                            "linear_total": 7077888,
                            "nnz": 382415,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 146432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 215040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 361472,
                            "linear_total": 7077888,
                            "nnz": 366316,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 402432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 850944,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1253376,
                            "linear_total": 7077888,
                            "nnz": 1259082,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 681984,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 826368,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1508352,
                            "linear_total": 7077888,
                            "nnz": 1514458,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 405504,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 923136,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1328640,
                            "linear_total": 7077888,
                            "nnz": 1334425,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 542720,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 880128,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1422848,
                            "linear_total": 7077888,
                            "nnz": 1428861,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 449536,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 645120,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1094656,
                            "linear_total": 7077888,
                            "nnz": 1100420,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 577536,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 525312,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1102848,
                            "linear_total": 7077888,
                            "nnz": 1108726,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 294912,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 333312,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 628224,
                            "linear_total": 7077888,
                            "nnz": 633497,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 320512,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 113664,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 434176,
                            "linear_total": 7077888,
                            "nnz": 439306,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 11493376,
                    "linear_sparsity": 86.46797839506173,
                    "linear_total": 84934656,
                    "nnz": 35398714,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "3": [
                            2,
                            3,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            8,
                            9,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            6,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 67.49225980035152
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 85.6109462422114,
            "fill_rate": 0.1353202160493826,
            "speedup": 2.86191312967017
        },
        {
            "annotate": "21",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.66887417218543,
                    "f1": 87.3881230572442
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v10-a32-l10-dl0-5--2021-01-21--00-54-13/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 0.5,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 17.326403350830077,
                    "eval_elapsed_time": 24.523588876239955
                },
                "speedup": 2.227490161916501,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 643072,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 634368,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1277440,
                            "linear_total": 7077888,
                            "nnz": 1283261,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 622592,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 916992,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1539584,
                            "linear_total": 7077888,
                            "nnz": 1545525,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 463872,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 112128,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 576000,
                            "linear_total": 7077888,
                            "nnz": 581449,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 278528,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 313344,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 591872,
                            "linear_total": 7077888,
                            "nnz": 597164,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1051648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1016832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2068480,
                            "linear_total": 7077888,
                            "nnz": 2075030,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1257472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1076736,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2334208,
                            "linear_total": 7077888,
                            "nnz": 2341053,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1315840,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1158144,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2473984,
                            "linear_total": 7077888,
                            "nnz": 2481074,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1004544,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1073664,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2078208,
                            "linear_total": 7077888,
                            "nnz": 2084891,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1004544,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 815616,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1820160,
                            "linear_total": 7077888,
                            "nnz": 1826675,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 925696,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 629760,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1555456,
                            "linear_total": 7077888,
                            "nnz": 1561658,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 899072,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 337920,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1236992,
                            "linear_total": 7077888,
                            "nnz": 1243132,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 523264,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 139776,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 663040,
                            "linear_total": 7077888,
                            "nnz": 668507,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 18215424,
                    "linear_sparsity": 78.55360243055556,
                    "linear_total": 84934656,
                    "nnz": 42128141,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 61.31241765669342
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.3881230572442,
            "fill_rate": 0.21446397569444442,
            "speedup": 2.227490161916501
        },
        {
            "annotate": "18",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.39451277199622,
                    "f1": 87.14755939306319
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v10-a32-l10-dl1--2021-01-21--00-53-40/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 17.057066314697266,
                    "eval_elapsed_time": 24.182081679347903
                },
                "speedup": 2.262663009764823,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 809984,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 297984,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1107968,
                            "linear_total": 7077888,
                            "nnz": 1113762,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 720896,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 483840,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1204736,
                            "linear_total": 7077888,
                            "nnz": 1210491,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 478208,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 73728,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 551936,
                            "linear_total": 7077888,
                            "nnz": 557392,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 312320,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 159744,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 472064,
                            "linear_total": 7077888,
                            "nnz": 477288,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1098752,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 619008,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1717760,
                            "linear_total": 7077888,
                            "nnz": 1724147,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1309696,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 657408,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1967104,
                            "linear_total": 7077888,
                            "nnz": 1973708,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1362944,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 705024,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2067968,
                            "linear_total": 7077888,
                            "nnz": 2074795,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1074176,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 668160,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1742336,
                            "linear_total": 7077888,
                            "nnz": 1748787,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1049600,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 516096,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1565696,
                            "linear_total": 7077888,
                            "nnz": 1572016,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 958464,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 384000,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1342464,
                            "linear_total": 7077888,
                            "nnz": 1348506,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 949248,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 204288,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1153536,
                            "linear_total": 7077888,
                            "nnz": 1159685,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 636928,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 92160,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 729088,
                            "linear_total": 7077888,
                            "nnz": 734684,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 15622656,
                    "linear_sparsity": 81.6062644675926,
                    "linear_total": 84934656,
                    "nnz": 39533983,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 63.694713643514845
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.14755939306319,
            "fill_rate": 0.18393735532407407,
            "speedup": 2.262663009764823
        },
        {
            "annotate": "16",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.67549668874172,
                    "f1": 86.51098653495667
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v10-a32-l10-dl2--2021-01-21--00-53-13/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 2.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 17.252509830474853,
                    "eval_elapsed_time": 24.480217491276562
                },
                "speedup": 2.2370306340702912,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 864256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 127488,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 991744,
                            "linear_total": 7077888,
                            "nnz": 997523,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 748544,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 216576,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 965120,
                            "linear_total": 7077888,
                            "nnz": 970733,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 502784,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 53760,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 556544,
                            "linear_total": 7077888,
                            "nnz": 562083,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 360448,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 81408,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 441856,
                            "linear_total": 7077888,
                            "nnz": 447157,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1163264,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 324096,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1487360,
                            "linear_total": 7077888,
                            "nnz": 1493555,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1389568,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 377856,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1767424,
                            "linear_total": 7077888,
                            "nnz": 1773910,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1449984,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 414720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1864704,
                            "linear_total": 7077888,
                            "nnz": 1871374,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1349632,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 364032,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1713664,
                            "linear_total": 7077888,
                            "nnz": 1720365,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1187840,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 293376,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1481216,
                            "linear_total": 7077888,
                            "nnz": 1487647,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 964608,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 225792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1190400,
                            "linear_total": 7077888,
                            "nnz": 1196403,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 1063936,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 127488,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1191424,
                            "linear_total": 7077888,
                            "nnz": 1197619,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 650240,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 58368,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 708608,
                            "linear_total": 7077888,
                            "nnz": 714182,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 14360064,
                    "linear_sparsity": 83.0928096064815,
                    "linear_total": 84934656,
                    "nnz": 38271273,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            1,
                            2
                        ],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 64.85429951512302
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.51098653495667,
            "fill_rate": 0.169071903935185,
            "speedup": 2.2370306340702912
        },
        {
            "annotate": "15",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.3349101229896,
                    "f1": 86.4116267700138
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v10-a32-l20-dl0-5--2021-01-21--00-55-44/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 0.5,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 14.760263885498047,
                    "eval_elapsed_time": 21.897933847736567
                },
                "speedup": 2.6147495264830645,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 519168,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 411648,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 930816,
                            "linear_total": 7077888,
                            "nnz": 936364,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 536576,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 592896,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1129472,
                            "linear_total": 7077888,
                            "nnz": 1135106,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 356352,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 87552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 443904,
                            "linear_total": 7077888,
                            "nnz": 449209,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 226304,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 199680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 425984,
                            "linear_total": 7077888,
                            "nnz": 431106,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 667648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 698880,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1366528,
                            "linear_total": 7077888,
                            "nnz": 1372487,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 967680,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 714240,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1681920,
                            "linear_total": 7077888,
                            "nnz": 1688273,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 835584,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 834048,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1669632,
                            "linear_total": 7077888,
                            "nnz": 1675967,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 668672,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 743424,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1412096,
                            "linear_total": 7077888,
                            "nnz": 1418052,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 653312,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 568320,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1221632,
                            "linear_total": 7077888,
                            "nnz": 1227506,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 787456,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 450048,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1237504,
                            "linear_total": 7077888,
                            "nnz": 1243493,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 493568,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 264192,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 757760,
                            "linear_total": 7077888,
                            "nnz": 763404,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 424960,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 101376,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 526336,
                            "linear_total": 7077888,
                            "nnz": 531586,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 12803584,
                    "linear_sparsity": 84.92537133487654,
                    "linear_total": 84934656,
                    "nnz": 36711275,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 66.28689420474849
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.4116267700138,
            "fill_rate": 0.15074628665123457,
            "speedup": 2.6147495264830645
        },
        {
            "annotate": "15",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.37275307473983,
                    "f1": 86.39441106336629
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v10-a32-l20-dl0-5--2021-01-21--00-55-44/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 0.5,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 14.746898548126222,
                    "eval_elapsed_time": 21.86237431317568
                },
                "speedup": 2.61711931355729,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 519168,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 411648,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 930816,
                            "linear_total": 7077888,
                            "nnz": 936364,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 536576,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 592896,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1129472,
                            "linear_total": 7077888,
                            "nnz": 1135106,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 356352,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 87552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 443904,
                            "linear_total": 7077888,
                            "nnz": 449209,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 226304,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 199680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 425984,
                            "linear_total": 7077888,
                            "nnz": 431106,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 667648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 698880,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1366528,
                            "linear_total": 7077888,
                            "nnz": 1372487,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 967680,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 714240,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1681920,
                            "linear_total": 7077888,
                            "nnz": 1688273,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 835584,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 834048,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1669632,
                            "linear_total": 7077888,
                            "nnz": 1675967,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 668672,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 743424,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1412096,
                            "linear_total": 7077888,
                            "nnz": 1418052,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 653312,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 568320,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1221632,
                            "linear_total": 7077888,
                            "nnz": 1227506,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 787456,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 450048,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1237504,
                            "linear_total": 7077888,
                            "nnz": 1243493,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 493568,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 264192,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 757760,
                            "linear_total": 7077888,
                            "nnz": 763404,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 424960,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 101376,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 526336,
                            "linear_total": 7077888,
                            "nnz": 531586,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 12803584,
                    "linear_sparsity": 84.92537133487654,
                    "linear_total": 84934656,
                    "nnz": 36711275,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 66.28689420474849
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.39441106336629,
            "fill_rate": 0.15074628665123457,
            "speedup": 2.61711931355729
        },
        {
            "annotate": "12",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.8240302743614,
                    "f1": 86.11992485005756
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v10-a32-l20-dl1--2021-01-21--00-55-15/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 14.268565601348877,
                    "eval_elapsed_time": 21.374552259687334
                },
                "speedup": 2.704854439028025,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 550912,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 181248,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 732160,
                            "linear_total": 7077888,
                            "nnz": 737654,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 535552,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 299520,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 835072,
                            "linear_total": 7077888,
                            "nnz": 840515,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 364544,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 58368,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 422912,
                            "linear_total": 7077888,
                            "nnz": 428102,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 239616,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 96768,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 336384,
                            "linear_total": 7077888,
                            "nnz": 341471,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 721920,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 407040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1128960,
                            "linear_total": 7077888,
                            "nnz": 1134729,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1111040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 440832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1551872,
                            "linear_total": 7077888,
                            "nnz": 1558207,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 892928,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 496128,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1389056,
                            "linear_total": 7077888,
                            "nnz": 1395267,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 663552,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 433152,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1096704,
                            "linear_total": 7077888,
                            "nnz": 1102458,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 662528,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 337920,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1000448,
                            "linear_total": 7077888,
                            "nnz": 1006172,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 801792,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 268800,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1070592,
                            "linear_total": 7077888,
                            "nnz": 1076463,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 645120,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 158208,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 803328,
                            "linear_total": 7077888,
                            "nnz": 809127,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 424960,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 73728,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 498688,
                            "linear_total": 7077888,
                            "nnz": 503952,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 10866176,
                    "linear_sparsity": 87.20642843364197,
                    "linear_total": 84934656,
                    "nnz": 34772839,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2,
                            4,
                            6,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 68.06702028169144
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.11992485005756,
            "fill_rate": 0.1279357156635803,
            "speedup": 2.704854439028025
        },
        {
            "annotate": "11",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 76.9914853358562,
                    "f1": 85.26341062121247
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v10-a32-l20-dl2--2021-01-21--00-54-43/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 2.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 14.846498733520509,
                    "eval_elapsed_time": 21.962527931667864
                },
                "speedup": 2.599561936999493,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 598016,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 76800,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 674816,
                            "linear_total": 7077888,
                            "nnz": 680306,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 621568,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 129024,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 750592,
                            "linear_total": 7077888,
                            "nnz": 756020,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 395264,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 36864,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 432128,
                            "linear_total": 7077888,
                            "nnz": 437432,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 238592,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 52224,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 290816,
                            "linear_total": 7077888,
                            "nnz": 295682,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 937984,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 199680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1137664,
                            "linear_total": 7077888,
                            "nnz": 1143554,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1193984,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 264192,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1458176,
                            "linear_total": 7077888,
                            "nnz": 1464428,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1057792,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 278016,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1335808,
                            "linear_total": 7077888,
                            "nnz": 1342037,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 614400,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 228864,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 843264,
                            "linear_total": 7077888,
                            "nnz": 848853,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 759808,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 188928,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 948736,
                            "linear_total": 7077888,
                            "nnz": 954619,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 830464,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 148992,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 979456,
                            "linear_total": 7077888,
                            "nnz": 985313,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 753664,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 79872,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 833536,
                            "linear_total": 7077888,
                            "nnz": 839444,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 432128,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 46080,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 478208,
                            "linear_total": 7077888,
                            "nnz": 483454,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 10163200,
                    "linear_sparsity": 88.03409529320987,
                    "linear_total": 84934656,
                    "nnz": 34069864,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            11,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2,
                            11,
                            4
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 68.71258409134985
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 85.26341062121247,
            "fill_rate": 0.11965904706790131,
            "speedup": 2.599561936999493
        },
        {
            "annotate": "15",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.06054872280038,
                    "f1": 86.20063710644014
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v10-a32-l30-dl0-25--2021-01-23--20-20-19/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 30.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 14.394198833465577,
                    "eval_elapsed_time": 21.72890411503613
                },
                "speedup": 2.681246344578876,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 455680,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 597504,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1053184,
                            "linear_total": 7077888,
                            "nnz": 1058789,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 364544,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 854016,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1218560,
                            "linear_total": 7077888,
                            "nnz": 1224172,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 286720,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 118272,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 404992,
                            "linear_total": 7077888,
                            "nnz": 410093,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 162816,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 276480,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 439296,
                            "linear_total": 7077888,
                            "nnz": 444244,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 529408,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 973824,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1503232,
                            "linear_total": 7077888,
                            "nnz": 1509178,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 749568,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 964608,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1714176,
                            "linear_total": 7077888,
                            "nnz": 1720436,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 578560,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1047552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1626112,
                            "linear_total": 7077888,
                            "nnz": 1632266,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 600064,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 992256,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1592320,
                            "linear_total": 7077888,
                            "nnz": 1598438,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 546816,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 775680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1322496,
                            "linear_total": 7077888,
                            "nnz": 1328505,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 686080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 615936,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1302016,
                            "linear_total": 7077888,
                            "nnz": 1308113,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 335872,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 342528,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 678400,
                            "linear_total": 7077888,
                            "nnz": 683743,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 358400,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 135168,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 493568,
                            "linear_total": 7077888,
                            "nnz": 498776,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 13348352,
                    "linear_sparsity": 84.28397472993827,
                    "linear_total": 84934656,
                    "nnz": 37255475,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 65.78713841653968
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.20063710644014,
            "fill_rate": 0.1571602527006174,
            "speedup": 2.681246344578876
        },
        {
            "annotate": "12",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.11447492904446,
                    "f1": 85.59611837921153
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v10-a32-l30-dl0-5--2021-01-23--20-19-50/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 0.5,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 30.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 13.499527885437011,
                    "eval_elapsed_time": 20.856850353069603
                },
                "speedup": 2.8589439077351635,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 451584,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 290304,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 741888,
                            "linear_total": 7077888,
                            "nnz": 747293,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 495616,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 459264,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 954880,
                            "linear_total": 7077888,
                            "nnz": 960395,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 296960,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 73728,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 370688,
                            "linear_total": 7077888,
                            "nnz": 375792,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 194560,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 153600,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 348160,
                            "linear_total": 7077888,
                            "nnz": 353092,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 583680,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 557568,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1141248,
                            "linear_total": 7077888,
                            "nnz": 1147019,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 789504,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 583680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1373184,
                            "linear_total": 7077888,
                            "nnz": 1379228,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 582656,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 665088,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1247744,
                            "linear_total": 7077888,
                            "nnz": 1253617,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 548864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 614400,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1163264,
                            "linear_total": 7077888,
                            "nnz": 1169040,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 578560,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 463872,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1042432,
                            "linear_total": 7077888,
                            "nnz": 1048302,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 715776,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 370176,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1085952,
                            "linear_total": 7077888,
                            "nnz": 1091889,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 375808,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 235008,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 610816,
                            "linear_total": 7077888,
                            "nnz": 616217,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 347136,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 89088,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 436224,
                            "linear_total": 7077888,
                            "nnz": 441306,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 10516480,
                    "linear_sparsity": 87.61815200617285,
                    "linear_total": 84934656,
                    "nnz": 34421912,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            3,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 68.3892874619354
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 85.59611837921153,
            "fill_rate": 0.12381847993827155,
            "speedup": 2.8589439077351635
        },
        {
            "annotate": "12",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.37937559129612,
                    "f1": 85.69020560735045
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v10-a32-l30-dl0-5--2021-01-23--20-19-50/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 0.5,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 30.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 13.492438529968261,
                    "eval_elapsed_time": 20.86975116888061
                },
                "speedup": 2.860446087610368,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 451584,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 290304,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 741888,
                            "linear_total": 7077888,
                            "nnz": 747293,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 495616,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 459264,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 954880,
                            "linear_total": 7077888,
                            "nnz": 960395,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 296960,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 73728,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 370688,
                            "linear_total": 7077888,
                            "nnz": 375792,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 194560,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 153600,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 348160,
                            "linear_total": 7077888,
                            "nnz": 353092,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 583680,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 557568,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1141248,
                            "linear_total": 7077888,
                            "nnz": 1147019,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 789504,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 583680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1373184,
                            "linear_total": 7077888,
                            "nnz": 1379228,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 582656,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 665088,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1247744,
                            "linear_total": 7077888,
                            "nnz": 1253617,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 548864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 614400,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1163264,
                            "linear_total": 7077888,
                            "nnz": 1169040,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 578560,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 463872,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1042432,
                            "linear_total": 7077888,
                            "nnz": 1048302,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 715776,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 370176,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1085952,
                            "linear_total": 7077888,
                            "nnz": 1091889,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 375808,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 235008,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 610816,
                            "linear_total": 7077888,
                            "nnz": 616217,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 347136,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 89088,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 436224,
                            "linear_total": 7077888,
                            "nnz": 441306,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 10516480,
                    "linear_sparsity": 87.61815200617285,
                    "linear_total": 84934656,
                    "nnz": 34421912,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            3,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 68.3892874619354
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 85.69020560735045,
            "fill_rate": 0.12381847993827155,
            "speedup": 2.860446087610368
        },
        {
            "annotate": "29",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.48249763481552,
                    "f1": 88.07285498416482
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v10-a32-l5-dl0-5--2021-01-21--00-52-45/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 0.5,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 5.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 20.669778549194337,
                    "eval_elapsed_time": 27.982159624807537
                },
                "speedup": 1.8671894773093938,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 978944,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1107456,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2086400,
                            "linear_total": 7077888,
                            "nnz": 2092881,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 721920,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1273344,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1995264,
                            "linear_total": 7077888,
                            "nnz": 2001533,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 615424,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 162816,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 778240,
                            "linear_total": 7077888,
                            "nnz": 783978,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 403456,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 477696,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 881152,
                            "linear_total": 7077888,
                            "nnz": 886839,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1232896,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1505280,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2738176,
                            "linear_total": 7077888,
                            "nnz": 2745300,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1455104,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1543680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2998784,
                            "linear_total": 7077888,
                            "nnz": 3006029,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1598464,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1609728,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3208192,
                            "linear_total": 7077888,
                            "nnz": 3215768,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1596416,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1508352,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3104768,
                            "linear_total": 7077888,
                            "nnz": 3112278,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1373184,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1125888,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2499072,
                            "linear_total": 7077888,
                            "nnz": 2506109,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 1165312,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 837120,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2002432,
                            "linear_total": 7077888,
                            "nnz": 2008993,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 1163264,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 468480,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1631744,
                            "linear_total": 7077888,
                            "nnz": 1638257,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 740352,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 207360,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 947712,
                            "linear_total": 7077888,
                            "nnz": 953447,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 24871936,
                    "linear_sparsity": 70.71638695987654,
                    "linear_total": 84934656,
                    "nnz": 48790134,
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            5,
                            6,
                            7
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            6,
                            7,
                            8,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            2
                        ],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            2,
                            3,
                            7
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 55.19450225287742
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.07285498416482,
            "fill_rate": 0.2928361304012346,
            "speedup": 1.8671894773093938
        },
        {
            "annotate": "29",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.70009460737937,
                    "f1": 88.04831949879843
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v10-a32-l5-dl0-5--2021-01-21--00-52-45/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 0.5,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 5.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 20.71169916152954,
                    "eval_elapsed_time": 28.054355942178518
                },
                "speedup": 1.863410273796239,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 978944,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1107456,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2086400,
                            "linear_total": 7077888,
                            "nnz": 2092881,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 721920,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1273344,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1995264,
                            "linear_total": 7077888,
                            "nnz": 2001533,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 615424,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 162816,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 778240,
                            "linear_total": 7077888,
                            "nnz": 783978,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 403456,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 477696,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 881152,
                            "linear_total": 7077888,
                            "nnz": 886839,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1232896,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1505280,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2738176,
                            "linear_total": 7077888,
                            "nnz": 2745300,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1455104,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1543680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2998784,
                            "linear_total": 7077888,
                            "nnz": 3006029,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1598464,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1609728,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3208192,
                            "linear_total": 7077888,
                            "nnz": 3215768,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1596416,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1508352,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3104768,
                            "linear_total": 7077888,
                            "nnz": 3112278,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1373184,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1125888,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2499072,
                            "linear_total": 7077888,
                            "nnz": 2506109,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 1165312,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 837120,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2002432,
                            "linear_total": 7077888,
                            "nnz": 2008993,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 1163264,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 468480,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1631744,
                            "linear_total": 7077888,
                            "nnz": 1638257,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 740352,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 207360,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 947712,
                            "linear_total": 7077888,
                            "nnz": 953447,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 24871936,
                    "linear_sparsity": 70.71638695987654,
                    "linear_total": 84934656,
                    "nnz": 48790134,
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            5,
                            6,
                            7
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            6,
                            7,
                            8,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            2
                        ],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            2,
                            3,
                            7
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 55.19450225287742
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.04831949879843,
            "fill_rate": 0.2928361304012346,
            "speedup": 1.863410273796239
        },
        {
            "annotate": "29",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.79470198675497,
                    "f1": 88.10958975740277
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v10-a32-l5-dl0-5--2021-01-21--00-52-45/checkpoint-95000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 0.5,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 5.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 20.632953029632567,
                    "eval_elapsed_time": 27.97396031860262
                },
                "speedup": 1.8705220212512832,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 978944,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1107456,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2086400,
                            "linear_total": 7077888,
                            "nnz": 2092881,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 721920,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1273344,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1995264,
                            "linear_total": 7077888,
                            "nnz": 2001533,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 615424,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 162816,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 778240,
                            "linear_total": 7077888,
                            "nnz": 783978,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 403456,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 477696,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 881152,
                            "linear_total": 7077888,
                            "nnz": 886839,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1232896,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1505280,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2738176,
                            "linear_total": 7077888,
                            "nnz": 2745300,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1455104,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1543680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2998784,
                            "linear_total": 7077888,
                            "nnz": 3006029,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1598464,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1609728,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3208192,
                            "linear_total": 7077888,
                            "nnz": 3215768,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1596416,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1508352,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3104768,
                            "linear_total": 7077888,
                            "nnz": 3112278,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1373184,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1125888,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2499072,
                            "linear_total": 7077888,
                            "nnz": 2506109,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 1165312,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 837120,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2002432,
                            "linear_total": 7077888,
                            "nnz": 2008993,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 1163264,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 468480,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1631744,
                            "linear_total": 7077888,
                            "nnz": 1638257,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 740352,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 207360,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 947712,
                            "linear_total": 7077888,
                            "nnz": 953447,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 24871936,
                    "linear_sparsity": 70.71638695987654,
                    "linear_total": 84934656,
                    "nnz": 48790134,
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            5,
                            6,
                            7
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            6,
                            7,
                            8,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            2
                        ],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            2,
                            3,
                            7
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 55.19450225287742
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.10958975740277,
            "fill_rate": 0.2928361304012346,
            "speedup": 1.8705220212512832
        },
        {
            "annotate": "24",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.58656575212866,
                    "f1": 88.06903108265608
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v10-a32-l5-dl1--2021-01-21--00-52-16/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 5.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 19.595643711090087,
                    "eval_elapsed_time": 26.718373194802552
                },
                "speedup": 1.9695394330694393,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1055744,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 526848,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1582592,
                            "linear_total": 7077888,
                            "nnz": 1588759,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 809984,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 752640,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1562624,
                            "linear_total": 7077888,
                            "nnz": 1568650,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 652288,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 98304,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 750592,
                            "linear_total": 7077888,
                            "nnz": 756384,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 419840,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 262656,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 682496,
                            "linear_total": 7077888,
                            "nnz": 688011,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1316864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 873984,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2190848,
                            "linear_total": 7077888,
                            "nnz": 2197625,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1468416,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 952320,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2420736,
                            "linear_total": 7077888,
                            "nnz": 2427596,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1651712,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1046016,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2697728,
                            "linear_total": 7077888,
                            "nnz": 2705001,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1616896,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 986112,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2603008,
                            "linear_total": 7077888,
                            "nnz": 2610178,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1361920,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 740352,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2102272,
                            "linear_total": 7077888,
                            "nnz": 2109058,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 1265664,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 559104,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1824768,
                            "linear_total": 7077888,
                            "nnz": 1831244,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 1212416,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 293376,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1505792,
                            "linear_total": 7077888,
                            "nnz": 1512127,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 749568,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 113664,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 863232,
                            "linear_total": 7077888,
                            "nnz": 868874,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 20786688,
                    "linear_sparsity": 75.52625868055556,
                    "linear_total": 84934656,
                    "nnz": 44702229,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            7
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            6,
                            7,
                            8,
                            11
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            2
                        ],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            2,
                            3,
                            7
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 58.94855257518133
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.06903108265608,
            "fill_rate": 0.24473741319444442,
            "speedup": 1.9695394330694393
        },
        {
            "annotate": "22",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.10406811731315,
                    "f1": 87.56487698206614
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v10-a32-l5-dl2--2021-01-21--00-51-49/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 2.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 5.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 19.459814723968506,
                    "eval_elapsed_time": 26.6199238197878
                },
                "speedup": 1.9832867657180042,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1210368,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 210432,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1420800,
                            "linear_total": 7077888,
                            "nnz": 1426953,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 977920,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 403968,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1381888,
                            "linear_total": 7077888,
                            "nnz": 1387879,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 712704,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 69120,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 781824,
                            "linear_total": 7077888,
                            "nnz": 787725,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 443392,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 136704,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 580096,
                            "linear_total": 7077888,
                            "nnz": 585529,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1500160,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 513024,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2013184,
                            "linear_total": 7077888,
                            "nnz": 2019886,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1526784,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 588288,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2115072,
                            "linear_total": 7077888,
                            "nnz": 2121727,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1734656,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 660480,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2395136,
                            "linear_total": 7077888,
                            "nnz": 2402190,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1659904,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 551424,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2211328,
                            "linear_total": 7077888,
                            "nnz": 2218215,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1486848,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 456192,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1943040,
                            "linear_total": 7077888,
                            "nnz": 1949865,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 1254400,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 336384,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1590784,
                            "linear_total": 7077888,
                            "nnz": 1597115,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 1267712,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 173568,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1441280,
                            "linear_total": 7077888,
                            "nnz": 1447569,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 760832,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 76800,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 837632,
                            "linear_total": 7077888,
                            "nnz": 843250,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 18712064,
                    "linear_sparsity": 77.96887056327161,
                    "linear_total": 84934656,
                    "nnz": 42626625,
                    "pruned_heads": {
                        "0": [
                            0,
                            5,
                            6,
                            7
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            6,
                            7
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            7
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            6,
                            7,
                            8,
                            11
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            3
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 60.85464429335368
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.56487698206614,
            "fill_rate": 0.22031129436728392,
            "speedup": 1.9832867657180042
        }
    ],
    "Block/struct method, bs= 32x32, v=1, s=b, t=l": [
        {
            "annotate": "15",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.77010406811732,
                    "f1": 86.63938864881486
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test4/hp_od-__data_2to__devel_data__nn_pruning__output__squad4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad4___dpm-sigmoied_threshold:1d_alt_ap--17cd29ad8a563746/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20
                },
                "speed": {
                    "cuda_eval_elapsed_time": 16.665313415527343,
                    "eval_elapsed_time": 23.629751751199365
                },
                "speedup": 2.3158516160525413,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 677888,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 310272,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 988160,
                            "linear_total": 7077888,
                            "nnz": 993834,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 689152,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 436224,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1125376,
                            "linear_total": 7077888,
                            "nnz": 1131132,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 434176,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 121344,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 555520,
                            "linear_total": 7077888,
                            "nnz": 560943,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 334848,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 156672,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 491520,
                            "linear_total": 7077888,
                            "nnz": 496838,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1087488,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 543744,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1631232,
                            "linear_total": 7077888,
                            "nnz": 1637570,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1189888,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 565248,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1755136,
                            "linear_total": 7077888,
                            "nnz": 1761552,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1104896,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 589824,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1694720,
                            "linear_total": 7077888,
                            "nnz": 1701216,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 818176,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 514560,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1332736,
                            "linear_total": 7077888,
                            "nnz": 1338767,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 882688,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 442368,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1325056,
                            "linear_total": 7077888,
                            "nnz": 1331200,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 846848,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 322560,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1169408,
                            "linear_total": 7077888,
                            "nnz": 1175442,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 732160,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 167424,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 899584,
                            "linear_total": 7077888,
                            "nnz": 905581,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 449536,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 84480,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 534016,
                            "linear_total": 7077888,
                            "nnz": 539287,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 13502464,
                    "linear_sparsity": 84.10252700617285,
                    "linear_total": 84934656,
                    "nnz": 37412084,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            10,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 65.64331950026698
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad4/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad4/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad4/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.63938864881486,
            "fill_rate": 0.15897472993827155,
            "speedup": 2.3158516160525413
        },
        {
            "annotate": "21",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.51087984862819,
                    "f1": 87.86127253902632
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test4/hp_od-__data_2to__devel_data__nn_pruning__output__squad4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad4___dpm-sigmoied_threshold:1d_alt_ap--754f92d6579864ca/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 18.421675689697267,
                    "eval_elapsed_time": 25.371724773198366
                },
                "speedup": 2.095053330406629,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 943104,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 528384,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1471488,
                            "linear_total": 7077888,
                            "nnz": 1477560,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 755712,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 744960,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1500672,
                            "linear_total": 7077888,
                            "nnz": 1506629,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 576512,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 173568,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 750080,
                            "linear_total": 7077888,
                            "nnz": 755729,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 439296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 253440,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 692736,
                            "linear_total": 7077888,
                            "nnz": 698309,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1308672,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 826368,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2135040,
                            "linear_total": 7077888,
                            "nnz": 2141754,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1400832,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 860160,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2260992,
                            "linear_total": 7077888,
                            "nnz": 2267792,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1519616,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 883200,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2402816,
                            "linear_total": 7077888,
                            "nnz": 2409823,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1276928,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 783360,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2060288,
                            "linear_total": 7077888,
                            "nnz": 2067070,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1248256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 629760,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1878016,
                            "linear_total": 7077888,
                            "nnz": 1884730,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 1039360,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 465408,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1504768,
                            "linear_total": 7077888,
                            "nnz": 1510991,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 960512,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 236544,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1197056,
                            "linear_total": 7077888,
                            "nnz": 1203194,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 608256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 118272,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 726528,
                            "linear_total": 7077888,
                            "nnz": 732141,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 18580480,
                    "linear_sparsity": 78.12379436728395,
                    "linear_total": 84934656,
                    "nnz": 42494444,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            2,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            1,
                            2
                        ],
                        "5": [
                            1,
                            2,
                            6
                        ],
                        "6": [
                            2,
                            3
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 60.97603021735447
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad4/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad4/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad4/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.86127253902632,
            "fill_rate": 0.2187620563271605,
            "speedup": 2.095053330406629
        },
        {
            "annotate": "22",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.5771050141911,
                    "f1": 87.9662592155432
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test4/hp_od-__data_2to__devel_data__nn_pruning__output__squad4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad4___dpm-sigmoied_threshold:1d_alt_ap--754f92d6579864ca/checkpoint-95000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 18.801320152282717,
                    "eval_elapsed_time": 25.66945153940469
                },
                "speedup": 2.0527490991469155,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 961536,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 543744,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1505280,
                            "linear_total": 7077888,
                            "nnz": 1511394,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 749568,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 764928,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1514496,
                            "linear_total": 7077888,
                            "nnz": 1520466,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 601088,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 176640,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 777728,
                            "linear_total": 7077888,
                            "nnz": 783379,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 449536,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 259584,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 709120,
                            "linear_total": 7077888,
                            "nnz": 714729,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1303552,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 835584,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2139136,
                            "linear_total": 7077888,
                            "nnz": 2145856,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1416192,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 866304,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2282496,
                            "linear_total": 7077888,
                            "nnz": 2289300,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1542144,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 893952,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2436096,
                            "linear_total": 7077888,
                            "nnz": 2443110,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1292288,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 797184,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2089472,
                            "linear_total": 7077888,
                            "nnz": 2096263,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1261568,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 637440,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1899008,
                            "linear_total": 7077888,
                            "nnz": 1905759,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 1064960,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 473088,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1538048,
                            "linear_total": 7077888,
                            "nnz": 1544276,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 1018880,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 241152,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1260032,
                            "linear_total": 7077888,
                            "nnz": 1266205,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 619520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 119808,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 739328,
                            "linear_total": 7077888,
                            "nnz": 744974,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 18890240,
                    "linear_sparsity": 77.75909047067901,
                    "linear_total": 84934656,
                    "nnz": 42804433,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            2,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            1,
                            2
                        ],
                        "5": [
                            1,
                            2,
                            6
                        ],
                        "6": [
                            2,
                            3
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 60.69135767595228
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad4/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad4/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad4/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.9662592155432,
            "fill_rate": 0.22240909529320985,
            "speedup": 2.0527490991469155
        },
        {
            "annotate": "12",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.67265846736045,
                    "f1": 85.80872913704097
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test4/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold--696f4785b3ba52e7/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 30
                },
                "speed": {
                    "cuda_eval_elapsed_time": 14.598422187805175,
                    "eval_elapsed_time": 21.40814032033086
                },
                "speedup": 2.6437372826229817,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 528384,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 222720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 751104,
                            "linear_total": 7077888,
                            "nnz": 756497,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 551936,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 344064,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 896000,
                            "linear_total": 7077888,
                            "nnz": 901504,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 399360,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 104448,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 503808,
                            "linear_total": 7077888,
                            "nnz": 509252,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 260096,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 129024,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 389120,
                            "linear_total": 7077888,
                            "nnz": 394228,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 867328,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 437760,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1305088,
                            "linear_total": 7077888,
                            "nnz": 1311069,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 987136,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 440832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1427968,
                            "linear_total": 7077888,
                            "nnz": 1434207,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 702464,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 443904,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1146368,
                            "linear_total": 7077888,
                            "nnz": 1152257,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 637952,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 427008,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1064960,
                            "linear_total": 7077888,
                            "nnz": 1070710,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 703488,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 334848,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1038336,
                            "linear_total": 7077888,
                            "nnz": 1044218,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 776192,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 265728,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1041920,
                            "linear_total": 7077888,
                            "nnz": 1047789,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 508928,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 138240,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 647168,
                            "linear_total": 7077888,
                            "nnz": 652730,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 386048,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 67584,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 453632,
                            "linear_total": 7077888,
                            "nnz": 458828,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 10665472,
                    "linear_sparsity": 87.44273244598766,
                    "linear_total": 84934656,
                    "nnz": 34572011,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            1,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            1,
                            2,
                            6,
                            7,
                            8,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "9": [
                            1,
                            2,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 68.25144688116664
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test4/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test4/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test4/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 85.80872913704097,
            "fill_rate": 0.1255726755401234,
            "speedup": 2.6437372826229817
        },
        {
            "annotate": "29",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 81.86376537369915,
                    "f1": 88.96065210705885
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test4/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold--6b3d26fc7262a898/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 5
                },
                "speed": {
                    "cuda_eval_elapsed_time": 21.24197428894043,
                    "eval_elapsed_time": 28.05219709314406
                },
                "speedup": 1.816892934733715,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1262592,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 964608,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2227200,
                            "linear_total": 7077888,
                            "nnz": 2233876,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 850944,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1202688,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2053632,
                            "linear_total": 7077888,
                            "nnz": 2060047,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 732160,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 256512,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 988672,
                            "linear_total": 7077888,
                            "nnz": 994567,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 600064,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 400896,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1000960,
                            "linear_total": 7077888,
                            "nnz": 1006917,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1501184,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1281024,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2782208,
                            "linear_total": 7077888,
                            "nnz": 2789378,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1546240,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1319424,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2865664,
                            "linear_total": 7077888,
                            "nnz": 2872859,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1752064,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1288704,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3040768,
                            "linear_total": 7077888,
                            "nnz": 3048199,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1638400,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1210368,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2848768,
                            "linear_total": 7077888,
                            "nnz": 2856084,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1496064,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 940032,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2436096,
                            "linear_total": 7077888,
                            "nnz": 2443172,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 1236992,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 662016,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1899008,
                            "linear_total": 7077888,
                            "nnz": 1905519,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 1231872,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 348672,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1580544,
                            "linear_total": 7077888,
                            "nnz": 1587011,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 806912,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 159744,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 966656,
                            "linear_total": 7077888,
                            "nnz": 972488,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 24690176,
                    "linear_sparsity": 70.9303867669753,
                    "linear_total": 84934656,
                    "nnz": 48608839,
                    "pruned_heads": {
                        "0": [
                            0,
                            4,
                            5,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7
                        ],
                        "11": [
                            8,
                            10,
                            5,
                            7
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            1
                        ],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            2,
                            3
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 55.3609910908475
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test4/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test4/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test4/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.96065210705885,
            "fill_rate": 0.29069613233024694,
            "speedup": 1.816892934733715
        },
        {
            "annotate": "38",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 82.94228949858089,
                    "f1": 89.74014850499854
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test4/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold:--d169c0ebde721c7/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 2.5
                },
                "speed": {
                    "cuda_eval_elapsed_time": 24.88833406829834,
                    "eval_elapsed_time": 31.90413049608469
                },
                "speedup": 1.550702144203815,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1664000,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1720320,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3384320,
                            "linear_total": 7077888,
                            "nnz": 3391936,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 1323008,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1815552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3138560,
                            "linear_total": 7077888,
                            "nnz": 3146014,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 966656,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 391680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1358336,
                            "linear_total": 7077888,
                            "nnz": 1364639,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 796672,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 638976,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1435648,
                            "linear_total": 7077888,
                            "nnz": 1442016,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1634304,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1961472,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3595776,
                            "linear_total": 7077888,
                            "nnz": 3603517,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1774592,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1981440,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3756032,
                            "linear_total": 7077888,
                            "nnz": 3763850,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 2001920,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1850880,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3852800,
                            "linear_total": 7077888,
                            "nnz": 3860821,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1760256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1752576,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3512832,
                            "linear_total": 7077888,
                            "nnz": 3520597,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1860608,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1350144,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3210752,
                            "linear_total": 7077888,
                            "nnz": 3218351,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 1329152,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1013760,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2342912,
                            "linear_total": 7077888,
                            "nnz": 2349684,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 1685504,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 511488,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2196992,
                            "linear_total": 7077888,
                            "nnz": 2204045,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 1032192,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 254976,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1287168,
                            "linear_total": 7077888,
                            "nnz": 1293286,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 33072128,
                    "linear_sparsity": 61.06168016975309,
                    "linear_total": 84934656,
                    "nnz": 56997478,
                    "pruned_heads": {
                        "0": [
                            0
                        ],
                        "1": [
                            0,
                            2,
                            3
                        ],
                        "10": [
                            1,
                            4,
                            7
                        ],
                        "11": [
                            8,
                            5,
                            7
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            2,
                            4
                        ],
                        "4": [],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            3
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [],
                        "9": [
                            1,
                            4,
                            5,
                            7
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 47.65744295515424
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test4/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test4/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test4/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 89.74014850499854,
            "fill_rate": 0.38938319830246915,
            "speedup": 1.550702144203815
        },
        {
            "annotate": "22",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.2620624408704,
                    "f1": 86.84661301619292
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test7/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test7___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test7___dpm-sigmoied_threshold--58e126daa38a2a47/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "sigmoied_threshold",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 24.72893617248535,
                    "eval_elapsed_time": 31.66312312334776
                },
                "speedup": 1.5606976675492068,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 735232,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1135282,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1870514,
                            "linear_total": 7077888,
                            "nnz": 1878535,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 577024,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1023832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1600856,
                            "linear_total": 7077888,
                            "nnz": 1608126,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 431616,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 467133,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 898749,
                            "linear_total": 7077888,
                            "nnz": 905169,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 393216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 547042,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 940258,
                            "linear_total": 7077888,
                            "nnz": 946936,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 783360,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1158540,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1941900,
                            "linear_total": 7077888,
                            "nnz": 1949830,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 784384,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1219713,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2004097,
                            "linear_total": 7077888,
                            "nnz": 2011949,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 884224,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1221421,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2105645,
                            "linear_total": 7077888,
                            "nnz": 2113786,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 836608,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1195904,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2032512,
                            "linear_total": 7077888,
                            "nnz": 2040562,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 827392,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1031874,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1859266,
                            "linear_total": 7077888,
                            "nnz": 1867230,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 715776,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 950014,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1665790,
                            "linear_total": 7077888,
                            "nnz": 1673098,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 793088,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 664201,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1457289,
                            "linear_total": 7077888,
                            "nnz": 1464667,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 467456,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 387427,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 854883,
                            "linear_total": 7077888,
                            "nnz": 861125,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 19231759,
                    "linear_sparsity": 77.35699429924105,
                    "linear_total": 84934656,
                    "nnz": 43159735,
                    "pruned_heads": {
                        "0": [
                            0
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            7,
                            8
                        ],
                        "11": [
                            8,
                            10,
                            5,
                            7
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            1,
                            2
                        ],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            3
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 60.36507279711698
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test7/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test7/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test7/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.84661301619292,
            "fill_rate": 0.22643005700758945,
            "speedup": 1.5606976675492068
        }
    ],
    "Block/struct method, bs= 32x32, v=1, s=b, t=l, nonorm=1": [
        {
            "annotate": "38",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 82.28949858088932,
                    "f1": 89.08071733584384
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test4/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold--5fcfb7ff678f0d71/checkpoint-100000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": 1,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 20000,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 5
                },
                "speed": {
                    "cuda_eval_elapsed_time": 23.795345176696777,
                    "eval_elapsed_time": 30.883679240942
                },
                "speedup": 1.6219303699434164,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1230848,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2323968,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3554816,
                            "linear_total": 7077888,
                            "nnz": 3562377,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 816128,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2049024,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2865152,
                            "linear_total": 7077888,
                            "nnz": 2871990,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 859136,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 935424,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1794560,
                            "linear_total": 7077888,
                            "nnz": 1801089,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 454656,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1512960,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1967616,
                            "linear_total": 7077888,
                            "nnz": 1974169,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1447936,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2105856,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3553792,
                            "linear_total": 7077888,
                            "nnz": 3561435,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1518592,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1918464,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3437056,
                            "linear_total": 7077888,
                            "nnz": 3444545,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1677312,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1692672,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3369984,
                            "linear_total": 7077888,
                            "nnz": 3377646,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1592832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2969088,
                            "linear_total": 7077888,
                            "nnz": 2976429,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1509376,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1403904,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2913280,
                            "linear_total": 7077888,
                            "nnz": 2920690,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 1168384,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1110528,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2278912,
                            "linear_total": 7077888,
                            "nnz": 2285587,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 1511424,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 837120,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2348544,
                            "linear_total": 7077888,
                            "nnz": 2355521,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 1092608,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 944640,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2037248,
                            "linear_total": 7077888,
                            "nnz": 2043911,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 33090048,
                    "linear_sparsity": 61.04058159722222,
                    "linear_total": 84934656,
                    "nnz": 57014111,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            7
                        ],
                        "11": [
                            0,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            1,
                            2
                        ],
                        "5": [
                            1,
                            2,
                            6
                        ],
                        "6": [
                            2,
                            3
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 47.64216835385825
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test4/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test4/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test4/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 89.08071733584384,
            "fill_rate": 0.3895941840277778,
            "speedup": 1.6219303699434164
        },
        {
            "annotate": "38",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 82.55439924314096,
                    "f1": 89.04891748061114
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test4/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold--5fcfb7ff678f0d71/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": 1,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 20000,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 5
                },
                "speed": {
                    "cuda_eval_elapsed_time": 23.309302703857423,
                    "eval_elapsed_time": 30.37249969970435
                },
                "speedup": 1.655750645813363,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1222656,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2322432,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3545088,
                            "linear_total": 7077888,
                            "nnz": 3552648,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 802816,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2038272,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2841088,
                            "linear_total": 7077888,
                            "nnz": 2847919,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 838656,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 935424,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1774080,
                            "linear_total": 7077888,
                            "nnz": 1780609,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 446464,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1503744,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1950208,
                            "linear_total": 7077888,
                            "nnz": 1956755,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1433600,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2096640,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3530240,
                            "linear_total": 7077888,
                            "nnz": 3537877,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1512448,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1915392,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3427840,
                            "linear_total": 7077888,
                            "nnz": 3435327,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1702912,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1684992,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3387904,
                            "linear_total": 7077888,
                            "nnz": 3395497,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1391616,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1589760,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2981376,
                            "linear_total": 7077888,
                            "nnz": 2988715,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1476608,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1399296,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2875904,
                            "linear_total": 7077888,
                            "nnz": 2883279,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 1180672,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1108992,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2289664,
                            "linear_total": 7077888,
                            "nnz": 2296338,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 1488896,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 830976,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2319872,
                            "linear_total": 7077888,
                            "nnz": 2326845,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 1105920,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 941568,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2047488,
                            "linear_total": 7077888,
                            "nnz": 2054149,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 32970752,
                    "linear_sparsity": 61.18103780864197,
                    "linear_total": 84934656,
                    "nnz": 56894680,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            7
                        ],
                        "11": [
                            0,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            1,
                            2
                        ],
                        "5": [
                            1,
                            2,
                            6
                        ],
                        "6": [
                            2,
                            3
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 47.75184555624996
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test4/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test4/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test4/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 89.04891748061114,
            "fill_rate": 0.3881896219135803,
            "speedup": 1.655750645813363
        },
        {
            "annotate": "23",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.86092715231788,
                    "f1": 87.96107619256047
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test4/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold--7eaf27127735c06f/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 2.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": 1,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 20000,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 19.622505153656007,
                    "eval_elapsed_time": 26.51508472301066
                },
                "speedup": 1.9668433109408714,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1031168,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1029120,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2060288,
                            "linear_total": 7077888,
                            "nnz": 2066750,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 781312,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 830976,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1612288,
                            "linear_total": 7077888,
                            "nnz": 1618301,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 642048,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 391680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1033728,
                            "linear_total": 7077888,
                            "nnz": 1039615,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 348160,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 591360,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 939520,
                            "linear_total": 7077888,
                            "nnz": 945377,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1391616,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 898560,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2290176,
                            "linear_total": 7077888,
                            "nnz": 2297001,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1443840,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 784896,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2228736,
                            "linear_total": 7077888,
                            "nnz": 2235487,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1585152,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 705024,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2290176,
                            "linear_total": 7077888,
                            "nnz": 2297067,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1073152,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 606720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1679872,
                            "linear_total": 7077888,
                            "nnz": 1686219,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1218560,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 572928,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1791488,
                            "linear_total": 7077888,
                            "nnz": 1798165,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 1105920,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 465408,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1571328,
                            "linear_total": 7077888,
                            "nnz": 1577551,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 1172480,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 356352,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1528832,
                            "linear_total": 7077888,
                            "nnz": 1535368,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 891904,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 362496,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1254400,
                            "linear_total": 7077888,
                            "nnz": 1260492,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 20280832,
                    "linear_sparsity": 76.12184124228395,
                    "linear_total": 84934656,
                    "nnz": 44196115,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            5,
                            6,
                            7
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            1,
                            2
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            2,
                            3
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            3
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 59.413332804864396
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test4/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test4/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test4/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.96107619256047,
            "fill_rate": 0.2387815875771605,
            "speedup": 1.9668433109408714
        },
        {
            "annotate": "21",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.4228949858089,
                    "f1": 86.81165607636312
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test4/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold:--63d7a49c946fed3/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": 1,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 20000,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20
                },
                "speed": {
                    "cuda_eval_elapsed_time": 17.33373427581787,
                    "eval_elapsed_time": 24.38529558479786
                },
                "speedup": 2.2265480935180695,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 704512,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1208832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1913344,
                            "linear_total": 7077888,
                            "nnz": 1919603,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 611328,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 992256,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1603584,
                            "linear_total": 7077888,
                            "nnz": 1609638,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 434176,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 442368,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 876544,
                            "linear_total": 7077888,
                            "nnz": 882208,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 649728,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 911872,
                            "linear_total": 7077888,
                            "nnz": 917479,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1036288,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1001472,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2037760,
                            "linear_total": 7077888,
                            "nnz": 2044492,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1167360,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 889344,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2056704,
                            "linear_total": 7077888,
                            "nnz": 2063395,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 885760,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 832512,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1718272,
                            "linear_total": 7077888,
                            "nnz": 1724606,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 773120,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 787968,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1561088,
                            "linear_total": 7077888,
                            "nnz": 1567265,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 815104,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 663552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1478656,
                            "linear_total": 7077888,
                            "nnz": 1484976,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 885760,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 576000,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1461760,
                            "linear_total": 7077888,
                            "nnz": 1467927,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 769024,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 436224,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1205248,
                            "linear_total": 7077888,
                            "nnz": 1211356,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 564224,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 463872,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1028096,
                            "linear_total": 7077888,
                            "nnz": 1033934,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 17852928,
                    "linear_sparsity": 78.98039641203704,
                    "linear_total": 84934656,
                    "nnz": 41765601,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            2,
                            3,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            1,
                            2,
                            6,
                            7,
                            8,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 61.64534941607825
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test4/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test4/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test4/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.81165607636312,
            "fill_rate": 0.21019603587962965,
            "speedup": 2.2265480935180695
        },
        {
            "annotate": "22",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.2620624408704,
                    "f1": 86.95749412557545
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test4/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold:--63d7a49c946fed3/checkpoint-80000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": 1,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 20000,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20
                },
                "speed": {
                    "cuda_eval_elapsed_time": 18.145860931396484,
                    "eval_elapsed_time": 25.041498971171677
                },
                "speedup": 2.126897872263199,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 741376,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1297920,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2039296,
                            "linear_total": 7077888,
                            "nnz": 2045645,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 668672,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1067520,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1736192,
                            "linear_total": 7077888,
                            "nnz": 1742295,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 448512,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 477696,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 926208,
                            "linear_total": 7077888,
                            "nnz": 931895,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 278528,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 671232,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 949760,
                            "linear_total": 7077888,
                            "nnz": 955445,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1107968,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1078272,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2186240,
                            "linear_total": 7077888,
                            "nnz": 2193022,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1221632,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 953856,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2175488,
                            "linear_total": 7077888,
                            "nnz": 2182221,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1090560,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 875520,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1966080,
                            "linear_total": 7077888,
                            "nnz": 1972666,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 837632,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 834048,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1671680,
                            "linear_total": 7077888,
                            "nnz": 1677887,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 891904,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 705024,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1596928,
                            "linear_total": 7077888,
                            "nnz": 1603275,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 940032,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 605184,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1545216,
                            "linear_total": 7077888,
                            "nnz": 1551466,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 857088,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 459264,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1316352,
                            "linear_total": 7077888,
                            "nnz": 1322475,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 630784,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 494592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1125376,
                            "linear_total": 7077888,
                            "nnz": 1131266,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 19234816,
                    "linear_sparsity": 77.35339506172839,
                    "linear_total": 84934656,
                    "nnz": 43148280,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            2,
                            3,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            8,
                            1,
                            2,
                            7
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 60.37559227994302
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test4/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test4/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test4/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.95749412557545,
            "fill_rate": 0.22646604938271608,
            "speedup": 2.126897872263199
        },
        {
            "annotate": "21",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.1608325449385,
                    "f1": 87.45888007303566
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test5/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test5___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test5___dpm-sigmoied_threshold--3006fe9afa215f73/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 2.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": 1,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 17.93759817504883,
                    "eval_elapsed_time": 24.918185566551983
                },
                "speedup": 2.151592015203899,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1049600,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 405504,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1455104,
                            "linear_total": 7077888,
                            "nnz": 1461256,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 737280,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 536064,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1273344,
                            "linear_total": 7077888,
                            "nnz": 1279165,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 622592,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 290304,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 912896,
                            "linear_total": 7077888,
                            "nnz": 918493,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 404480,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 423936,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 828416,
                            "linear_total": 7077888,
                            "nnz": 834036,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1270784,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 744960,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2015744,
                            "linear_total": 7077888,
                            "nnz": 2022405,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1457152,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 726528,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2183680,
                            "linear_total": 7077888,
                            "nnz": 2190393,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1575936,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 705024,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2280960,
                            "linear_total": 7077888,
                            "nnz": 2287883,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1094656,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 635904,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1730560,
                            "linear_total": 7077888,
                            "nnz": 1736926,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1110016,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 539136,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1649152,
                            "linear_total": 7077888,
                            "nnz": 1655551,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 1128448,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 423936,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1552384,
                            "linear_total": 7077888,
                            "nnz": 1558580,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 1105920,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 213504,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1319424,
                            "linear_total": 7077888,
                            "nnz": 1325707,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 644096,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 147456,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 791552,
                            "linear_total": 7077888,
                            "nnz": 797152,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 17993216,
                    "linear_sparsity": 78.81522472993827,
                    "linear_total": 84934656,
                    "nnz": 41906269,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            5,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            1,
                            2
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 61.51616961597579
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test5/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test5/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test5/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.45888007303566,
            "fill_rate": 0.2118477527006174,
            "speedup": 2.151592015203899
        },
        {
            "annotate": "25",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 81.10690633869442,
                    "f1": 88.04977607167375
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test5/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test5___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test5___dpm-sigmoied_threshold--35f0c1d86d528754/checkpoint-105000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": 1,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 18.656167499542235,
                    "eval_elapsed_time": 25.615012356080115
                },
                "speedup": 2.0687203310282287,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 847872,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 771072,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1618944,
                            "linear_total": 7077888,
                            "nnz": 1625078,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 686080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1009152,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1695232,
                            "linear_total": 7077888,
                            "nnz": 1701361,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 615424,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 539136,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1154560,
                            "linear_total": 7077888,
                            "nnz": 1160319,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 369664,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 769536,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1139200,
                            "linear_total": 7077888,
                            "nnz": 1144885,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1100800,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1207296,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2308096,
                            "linear_total": 7077888,
                            "nnz": 2314866,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1414144,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1201152,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2615296,
                            "linear_total": 7077888,
                            "nnz": 2622318,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1507328,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1156608,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2663936,
                            "linear_total": 7077888,
                            "nnz": 2671121,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1105920,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1056768,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2162688,
                            "linear_total": 7077888,
                            "nnz": 2169328,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1052672,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 960000,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2012672,
                            "linear_total": 7077888,
                            "nnz": 2019345,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 1116160,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 718848,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1835008,
                            "linear_total": 7077888,
                            "nnz": 1841396,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 1076224,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 390144,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1466368,
                            "linear_total": 7077888,
                            "nnz": 1472606,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 655360,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 279552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 934912,
                            "linear_total": 7077888,
                            "nnz": 940630,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 21606912,
                    "linear_sparsity": 74.560546875,
                    "linear_total": 84934656,
                    "nnz": 45521975,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            5,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            1,
                            2
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            10,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 58.19575432387478
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test5/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test5/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test5/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.04977607167375,
            "fill_rate": 0.25439453125,
            "speedup": 2.0687203310282287
        },
        {
            "annotate": "25",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 81.02175969725639,
                    "f1": 87.97543121375101
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test5/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test5___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test5___dpm-sigmoied_threshold--35f0c1d86d528754/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": 1,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 18.665786254882814,
                    "eval_elapsed_time": 25.75737725570798
                },
                "speedup": 2.067654288887784,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 855040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 771072,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1626112,
                            "linear_total": 7077888,
                            "nnz": 1632214,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 680960,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1009152,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1690112,
                            "linear_total": 7077888,
                            "nnz": 1696241,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 610304,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 539136,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1149440,
                            "linear_total": 7077888,
                            "nnz": 1155199,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 367616,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 766464,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1134080,
                            "linear_total": 7077888,
                            "nnz": 1139731,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1110016,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1204224,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2314240,
                            "linear_total": 7077888,
                            "nnz": 2320976,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1414144,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1199616,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2613760,
                            "linear_total": 7077888,
                            "nnz": 2620781,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1499136,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1153536,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2652672,
                            "linear_total": 7077888,
                            "nnz": 2659855,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1121280,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1053696,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2174976,
                            "linear_total": 7077888,
                            "nnz": 2181614,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1007616,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 958464,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1966080,
                            "linear_total": 7077888,
                            "nnz": 1972688,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 1094656,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 718848,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1813504,
                            "linear_total": 7077888,
                            "nnz": 1819892,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 1044480,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 390144,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1434624,
                            "linear_total": 7077888,
                            "nnz": 1440862,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 655360,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 279552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 934912,
                            "linear_total": 7077888,
                            "nnz": 940630,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 21504512,
                    "linear_sparsity": 74.68111014660495,
                    "linear_total": 84934656,
                    "nnz": 45419405,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            5,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            1,
                            2
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            10,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 58.289947545478185
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test5/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test5/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test5/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.97543121375101,
            "fill_rate": 0.25318889853395055,
            "speedup": 2.067654288887784
        },
        {
            "annotate": "27",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 81.13528855250709,
                    "f1": 88.07511972962917
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test6/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test6___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test6___dpm-sigmoied_threshold:--4b86dc18da73d79/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": 1,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 5000,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 19.85627264404297,
                    "eval_elapsed_time": 26.85605499520898
                },
                "speedup": 1.9436877050008528,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 918528,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1864704,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2783232,
                            "linear_total": 7077888,
                            "nnz": 2790110,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 671744,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1508352,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2180096,
                            "linear_total": 7077888,
                            "nnz": 2186454,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 574464,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 528384,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1102848,
                            "linear_total": 7077888,
                            "nnz": 1108888,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 95232,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 705024,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 800256,
                            "linear_total": 7077888,
                            "nnz": 805419,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1248256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1400832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2649088,
                            "linear_total": 7077888,
                            "nnz": 2656112,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1334272,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1277952,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2612224,
                            "linear_total": 7077888,
                            "nnz": 2619200,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1512448,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1105920,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2618368,
                            "linear_total": 7077888,
                            "nnz": 2625520,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1064960,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1058304,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2123264,
                            "linear_total": 7077888,
                            "nnz": 2129905,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1215488,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 903168,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2118656,
                            "linear_total": 7077888,
                            "nnz": 2125548,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 1078272,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 698880,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1777152,
                            "linear_total": 7077888,
                            "nnz": 1783527,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 948224,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 529920,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1478144,
                            "linear_total": 7077888,
                            "nnz": 1484473,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 863232,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 635904,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1499136,
                            "linear_total": 7077888,
                            "nnz": 1505566,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 23742464,
                    "linear_sparsity": 72.04619984567901,
                    "linear_total": 84934656,
                    "nnz": 47659444,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            2,
                            3,
                            4,
                            5,
                            8,
                            9,
                            10
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            1,
                            2
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            2,
                            3
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            10,
                            4,
                            5
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 56.23285005179295
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test6/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test6/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test6/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.07511972962917,
            "fill_rate": 0.27953800154320985,
            "speedup": 1.9436877050008528
        }
    ],
    "Block/struct method, bs= 32x32, v=1, s=b, t=l, nonorm=1, relu=1": [
        {
            "annotate": "15",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.6158940397351,
                    "f1": 85.58086718082839
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch4/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test_9_fullpatch4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test_9_fullpatch4_--6b7cbdb694e8fe5f/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": 1,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": 1,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "linear_min_parameters": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 30
                },
                "speed": {
                    "cuda_eval_elapsed_time": 15.16314505004883,
                    "eval_elapsed_time": 22.14020838122815
                },
                "speedup": 2.5452762522539354,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 607232,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 391680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 998912,
                            "linear_total": 7077888,
                            "nnz": 1004543,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 525312,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 488448,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1013760,
                            "linear_total": 7077888,
                            "nnz": 1019454,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 400384,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 228864,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 629248,
                            "linear_total": 7077888,
                            "nnz": 634581,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 268288,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 337920,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 606208,
                            "linear_total": 7077888,
                            "nnz": 611452,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 774144,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 685056,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1459200,
                            "linear_total": 7077888,
                            "nnz": 1465310,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1130496,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 683520,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1814016,
                            "linear_total": 7077888,
                            "nnz": 1820509,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 743424,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 668160,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1411584,
                            "linear_total": 7077888,
                            "nnz": 1417651,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 513024,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 591360,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1104384,
                            "linear_total": 7077888,
                            "nnz": 1110049,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 641024,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 551424,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1192448,
                            "linear_total": 7077888,
                            "nnz": 1198375,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 849920,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 436224,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1286144,
                            "linear_total": 7077888,
                            "nnz": 1292220,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 576512,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 225792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 802304,
                            "linear_total": 7077888,
                            "nnz": 808083,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 387072,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 129024,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 516096,
                            "linear_total": 7077888,
                            "nnz": 521396,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 12834304,
                    "linear_sparsity": 84.88920235339506,
                    "linear_total": 84934656,
                    "nnz": 36742345,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            4,
                            7,
                            8
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2,
                            6,
                            7,
                            8,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            6,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            4,
                            5,
                            7,
                            8
                        ],
                        "9": [
                            1,
                            2,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 66.25836165726659
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch4/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch4/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch4/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 85.58086718082839,
            "fill_rate": 0.15110797646604934,
            "speedup": 2.5452762522539354
        },
        {
            "annotate": "17",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.09839167455061,
                    "f1": 85.89097404430157
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch4/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test_9_fullpatch4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test_9_fullpatch4_--6b7cbdb694e8fe5f/checkpoint-65000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": 1,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": 1,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "linear_min_parameters": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 30
                },
                "speed": {
                    "cuda_eval_elapsed_time": 16.425722427368164,
                    "eval_elapsed_time": 23.391399671323597
                },
                "speedup": 2.349631389184198,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 709632,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 462336,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1171968,
                            "linear_total": 7077888,
                            "nnz": 1177805,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 603136,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 551424,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1154560,
                            "linear_total": 7077888,
                            "nnz": 1160391,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 531456,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 267264,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 798720,
                            "linear_total": 7077888,
                            "nnz": 804302,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 321536,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 450048,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 771584,
                            "linear_total": 7077888,
                            "nnz": 776965,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1000448,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 717312,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1717760,
                            "linear_total": 7077888,
                            "nnz": 1724211,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1271808,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 723456,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1995264,
                            "linear_total": 7077888,
                            "nnz": 2001847,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 873472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 721920,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1595392,
                            "linear_total": 7077888,
                            "nnz": 1601622,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 835584,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 632832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1468416,
                            "linear_total": 7077888,
                            "nnz": 1474524,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 783360,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 595968,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1379328,
                            "linear_total": 7077888,
                            "nnz": 1385444,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 1017856,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 483840,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1501696,
                            "linear_total": 7077888,
                            "nnz": 1507899,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 829440,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 242688,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1072128,
                            "linear_total": 7077888,
                            "nnz": 1078142,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 522240,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 133632,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 655872,
                            "linear_total": 7077888,
                            "nnz": 661271,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 15282688,
                    "linear_sparsity": 82.00653452932099,
                    "linear_total": 84934656,
                    "nnz": 39193145,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            6,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            4
                        ],
                        "9": [
                            1,
                            2,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 64.00771578122438
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch4/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch4/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch4/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 85.89097404430157,
            "fill_rate": 0.17993465470679015,
            "speedup": 2.349631389184198
        },
        {
            "annotate": "15",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.42667928098392,
                    "f1": 85.63059701238892
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch4/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test_9_fullpatch4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test_9_fullpatch4_--6b7cbdb694e8fe5f/checkpoint-95000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": 1,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": 1,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "linear_min_parameters": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 30
                },
                "speed": {
                    "cuda_eval_elapsed_time": 15.290082546234132,
                    "eval_elapsed_time": 22.249811742454767
                },
                "speedup": 2.5241454968388446,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 613376,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 400896,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1014272,
                            "linear_total": 7077888,
                            "nnz": 1019909,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 542720,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 494592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1037312,
                            "linear_total": 7077888,
                            "nnz": 1043010,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 417792,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 231936,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 649728,
                            "linear_total": 7077888,
                            "nnz": 655095,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 276480,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 345600,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 622080,
                            "linear_total": 7077888,
                            "nnz": 627329,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 825344,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 685056,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1510400,
                            "linear_total": 7077888,
                            "nnz": 1516542,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1183744,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 686592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1870336,
                            "linear_total": 7077888,
                            "nnz": 1876863,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 764928,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 674304,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1439232,
                            "linear_total": 7077888,
                            "nnz": 1445303,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 518144,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 595968,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1114112,
                            "linear_total": 7077888,
                            "nnz": 1119780,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 677888,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 552960,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1230848,
                            "linear_total": 7077888,
                            "nnz": 1236776,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 875520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 437760,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1313280,
                            "linear_total": 7077888,
                            "nnz": 1319357,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 623616,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 228864,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 852480,
                            "linear_total": 7077888,
                            "nnz": 858261,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 401408,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 130560,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 531968,
                            "linear_total": 7077888,
                            "nnz": 537333,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 13186048,
                    "linear_sparsity": 84.4750675154321,
                    "linear_total": 84934656,
                    "nnz": 37094280,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            1,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2,
                            6,
                            7,
                            8,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            6,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            4,
                            5,
                            7,
                            8
                        ],
                        "9": [
                            1,
                            2,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 65.9351687992672
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch4/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch4/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch4/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 85.63059701238892,
            "fill_rate": 0.155249324845679,
            "speedup": 2.5241454968388446
        },
        {
            "annotate": "27",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 81.30558183538317,
                    "f1": 88.32616013981377
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch4/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test_9_fullpatch4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test_9_fullpatch4_--6cb2db64e9a885f1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": 1,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": 1,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "linear_min_parameters": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 19.66401289367676,
                    "eval_elapsed_time": 26.647668646648526
                },
                "speedup": 1.9626916038984936,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 920576,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1188864,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2109440,
                            "linear_total": 7077888,
                            "nnz": 2115942,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 709632,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1187328,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1896960,
                            "linear_total": 7077888,
                            "nnz": 1903205,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 617472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 552960,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1170432,
                            "linear_total": 7077888,
                            "nnz": 1176200,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 423936,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 806400,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1230336,
                            "linear_total": 7077888,
                            "nnz": 1236333,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1210368,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1371648,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2582016,
                            "linear_total": 7077888,
                            "nnz": 2589021,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1433600,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1310208,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2743808,
                            "linear_total": 7077888,
                            "nnz": 2750901,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1558528,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1219584,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2778112,
                            "linear_total": 7077888,
                            "nnz": 2785338,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1038336,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1213440,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2251776,
                            "linear_total": 7077888,
                            "nnz": 2258486,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1107968,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 967680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2075648,
                            "linear_total": 7077888,
                            "nnz": 2082422,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 1108992,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 766464,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1875456,
                            "linear_total": 7077888,
                            "nnz": 1881875,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 1062912,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 399360,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1462272,
                            "linear_total": 7077888,
                            "nnz": 1468548,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 670720,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 274944,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 945664,
                            "linear_total": 7077888,
                            "nnz": 951443,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 23121920,
                    "linear_sparsity": 72.77681327160495,
                    "linear_total": 84934656,
                    "nnz": 47038436,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            1,
                            2
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            10,
                            2,
                            3
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 56.8031410156371
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch4/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch4/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch4/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.32616013981377,
            "fill_rate": 0.27223186728395055,
            "speedup": 1.9626916038984936
        },
        {
            "annotate": "18",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.60264900662251,
                    "f1": 86.94398255916529
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch4/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test_9_fullpatch4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test_9_fullpatch4_--75942d701c2bed7e/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": 1,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": 1,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "linear_min_parameters": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20
                },
                "speed": {
                    "cuda_eval_elapsed_time": 16.995676197052003,
                    "eval_elapsed_time": 23.85530902631581
                },
                "speedup": 2.2708359795690574,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 726016,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 569856,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1295872,
                            "linear_total": 7077888,
                            "nnz": 1301779,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 521216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 681984,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1203200,
                            "linear_total": 7077888,
                            "nnz": 1209020,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 500736,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 324096,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 824832,
                            "linear_total": 7077888,
                            "nnz": 830451,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 302080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 482304,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 784384,
                            "linear_total": 7077888,
                            "nnz": 789786,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 843776,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 832512,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1676288,
                            "linear_total": 7077888,
                            "nnz": 1682526,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1219584,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 877056,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2096640,
                            "linear_total": 7077888,
                            "nnz": 2103355,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1117184,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 850944,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1968128,
                            "linear_total": 7077888,
                            "nnz": 1974826,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 769024,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 827904,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1596928,
                            "linear_total": 7077888,
                            "nnz": 1603131,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 732160,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 700416,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1432576,
                            "linear_total": 7077888,
                            "nnz": 1438664,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 946176,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 532992,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1479168,
                            "linear_total": 7077888,
                            "nnz": 1485403,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 797696,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 285696,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1083392,
                            "linear_total": 7077888,
                            "nnz": 1089402,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 510976,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 139776,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 650752,
                            "linear_total": 7077888,
                            "nnz": 656283,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 16092160,
                    "linear_sparsity": 81.05348186728395,
                    "linear_total": 84934656,
                    "nnz": 40003348,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            4,
                            5,
                            7,
                            8
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            1,
                            2,
                            7
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            6,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 63.26368116366804
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch4/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch4/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch4/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.94398255916529,
            "fill_rate": 0.1894651813271605,
            "speedup": 2.2708359795690574
        }
    ],
    "Block/struct method, bs= 32x32, v=1, s=b, t=l, nonorm=1, relu=1, rewind=1": [
        {
            "annotate": "30",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 82.21381267738883,
                    "f1": 89.18874369381042
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch6/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test_9_fullpatch6___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test_9_fullpatch6_--5f772c87c5edbc85/checkpoint-100000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": 1,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": 1,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "linear_min_parameters": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10,
                    "rewind_model_name_or_path": "madlag/bert-base-uncased-squadv1-x1.96-f88.3-d27-hybrid-filled-opt-v1"
                },
                "speed": {
                    "cuda_eval_elapsed_time": 19.22297591018677,
                    "eval_elapsed_time": 26.312922549434006
                },
                "speedup": 2.00772207100977,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 734208,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2110464,
                            "linear_total": 7077888,
                            "nnz": 2116894,
                            "total": 7086912
                        },
                        "1": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 814080,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1797120,
                            "linear_total": 7077888,
                            "nnz": 1803218,
                            "total": 7086528
                        },
                        "10": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 505344,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1488384,
                            "linear_total": 7077888,
                            "nnz": 1494281,
                            "total": 7086528
                        },
                        "11": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 728064,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1907712,
                            "linear_total": 7077888,
                            "nnz": 1913946,
                            "total": 7086720
                        },
                        "2": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1021440,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2790912,
                            "linear_total": 7077888,
                            "nnz": 2797913,
                            "total": 7087296
                        },
                        "3": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 964608,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2734080,
                            "linear_total": 7077888,
                            "nnz": 2741044,
                            "total": 7087296
                        },
                        "4": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 835584,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2801664,
                            "linear_total": 7077888,
                            "nnz": 2808736,
                            "total": 7087488
                        },
                        "5": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 857088,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2233344,
                            "linear_total": 7077888,
                            "nnz": 2239854,
                            "total": 7086912
                        },
                        "6": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 740352,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2509824,
                            "linear_total": 7077888,
                            "nnz": 2516642,
                            "total": 7087296
                        },
                        "7": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 563712,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1939968,
                            "linear_total": 7077888,
                            "nnz": 1946287,
                            "total": 7086912
                        },
                        "8": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 282624,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2052096,
                            "linear_total": 7077888,
                            "nnz": 2058616,
                            "total": 7087296
                        },
                        "9": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 201216,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1380864,
                            "linear_total": 7077888,
                            "nnz": 1386755,
                            "total": 7086720
                        }
                    },
                    "linear_nnz": 25746432,
                    "linear_sparsity": 69.68677662037037,
                    "linear_total": 84934656,
                    "nnz": 49662908,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            1,
                            2
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            3,
                            10,
                            2
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108882626,
                    "total_sparsity": 54.38858353765275
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch6/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch6/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch6/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 89.18874369381042,
            "fill_rate": 0.3031322337962963,
            "speedup": 2.00772207100977
        },
        {
            "annotate": "30",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 82.06244087038789,
                    "f1": 89.0933430047216
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch6/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test_9_fullpatch6___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test_9_fullpatch6_--5f772c87c5edbc85/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": 1,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": 1,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "linear_min_parameters": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10,
                    "rewind_model_name_or_path": "madlag/bert-base-uncased-squadv1-x1.96-f88.3-d27-hybrid-filled-opt-v1"
                },
                "speed": {
                    "cuda_eval_elapsed_time": 18.779986827850344,
                    "eval_elapsed_time": 25.796454546041787
                },
                "speedup": 2.055080941171715,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 728064,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2104320,
                            "linear_total": 7077888,
                            "nnz": 2110746,
                            "total": 7086912
                        },
                        "1": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 807936,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1790976,
                            "linear_total": 7077888,
                            "nnz": 1797070,
                            "total": 7086528
                        },
                        "10": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 499200,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1482240,
                            "linear_total": 7077888,
                            "nnz": 1488133,
                            "total": 7086528
                        },
                        "11": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 715776,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1895424,
                            "linear_total": 7077888,
                            "nnz": 1901650,
                            "total": 7086720
                        },
                        "2": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1021440,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2790912,
                            "linear_total": 7077888,
                            "nnz": 2797913,
                            "total": 7087296
                        },
                        "3": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 952320,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2721792,
                            "linear_total": 7077888,
                            "nnz": 2728748,
                            "total": 7087296
                        },
                        "4": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 829440,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2795520,
                            "linear_total": 7077888,
                            "nnz": 2802588,
                            "total": 7087488
                        },
                        "5": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 849408,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2225664,
                            "linear_total": 7077888,
                            "nnz": 2232169,
                            "total": 7086912
                        },
                        "6": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 732672,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2502144,
                            "linear_total": 7077888,
                            "nnz": 2508957,
                            "total": 7087296
                        },
                        "7": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 554496,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1930752,
                            "linear_total": 7077888,
                            "nnz": 1937065,
                            "total": 7086912
                        },
                        "8": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 281088,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2050560,
                            "linear_total": 7077888,
                            "nnz": 2057079,
                            "total": 7087296
                        },
                        "9": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 199680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1379328,
                            "linear_total": 7077888,
                            "nnz": 1385218,
                            "total": 7086720
                        }
                    },
                    "linear_nnz": 25669632,
                    "linear_sparsity": 69.77719907407408,
                    "linear_total": 84934656,
                    "nnz": 49586058,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            1,
                            2
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            3,
                            10,
                            2
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108882626,
                    "total_sparsity": 54.45916412780125
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch6/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch6/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch6/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 89.0933430047216,
            "fill_rate": 0.3022280092592592,
            "speedup": 2.055080941171715
        }
    ],
    "Block/struct method, bs= 32x32, v=1, s=b, t=l, relu=1": [
        {
            "annotate": "16",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.4550614947966,
                    "f1": 85.61113138424535
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_9_gelupatch/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test_9_fullpatch___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test_9_fullpatch___--4520aaa044f7c325/checkpoint-105000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": 1,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": 0,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20
                },
                "speed": {
                    "cuda_eval_elapsed_time": 16.52789610671997,
                    "eval_elapsed_time": 23.538856061175466
                },
                "speedup": 2.3351062201848696,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 658432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 316416,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 974848,
                            "linear_total": 7077888,
                            "nnz": 980526,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 706560,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 408576,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1115136,
                            "linear_total": 7077888,
                            "nnz": 1120874,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 458752,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 136704,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 595456,
                            "linear_total": 7077888,
                            "nnz": 600921,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 351232,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 165888,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 517120,
                            "linear_total": 7077888,
                            "nnz": 522476,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1111040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 532992,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1644032,
                            "linear_total": 7077888,
                            "nnz": 1650363,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1268736,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 595968,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1864704,
                            "linear_total": 7077888,
                            "nnz": 1871268,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1108992,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 588288,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1697280,
                            "linear_total": 7077888,
                            "nnz": 1703807,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 827392,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 556032,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1383424,
                            "linear_total": 7077888,
                            "nnz": 1389482,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 898048,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 471552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1369600,
                            "linear_total": 7077888,
                            "nnz": 1375891,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 825344,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 356352,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1181696,
                            "linear_total": 7077888,
                            "nnz": 1187688,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 779264,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 175104,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 954368,
                            "linear_total": 7077888,
                            "nnz": 960402,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 436224,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 87552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 523776,
                            "linear_total": 7077888,
                            "nnz": 529113,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 13821440,
                    "linear_sparsity": 83.72697241512346,
                    "linear_total": 84934656,
                    "nnz": 37731533,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            1,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            10,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 65.34995954659642
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 78.4957426679281,
                    "f1": 86.42074759124448
                }
            },
            "f1": 85.61113138424535,
            "fill_rate": 0.16273027584876543,
            "speedup": 2.3351062201848696
        },
        {
            "annotate": "16",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.89025543992432,
                    "f1": 85.87492067079658
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_9_gelupatch/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test_9_fullpatch___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test_9_fullpatch___--4520aaa044f7c325/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": 1,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": 0,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20
                },
                "speed": {
                    "cuda_eval_elapsed_time": 16.544022068023683,
                    "eval_elapsed_time": 23.504416819661856
                },
                "speedup": 2.332830121156959,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 654336,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 316416,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 970752,
                            "linear_total": 7077888,
                            "nnz": 976366,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 700416,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 408576,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1108992,
                            "linear_total": 7077888,
                            "nnz": 1114730,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 459776,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 136704,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 596480,
                            "linear_total": 7077888,
                            "nnz": 601945,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 337920,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 165888,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 503808,
                            "linear_total": 7077888,
                            "nnz": 509100,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1081344,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 531456,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1612800,
                            "linear_total": 7077888,
                            "nnz": 1619130,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1241088,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 595968,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1837056,
                            "linear_total": 7077888,
                            "nnz": 1843620,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1072128,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 586752,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1658880,
                            "linear_total": 7077888,
                            "nnz": 1665310,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 806912,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 551424,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1358336,
                            "linear_total": 7077888,
                            "nnz": 1364391,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 901120,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 471552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1372672,
                            "linear_total": 7077888,
                            "nnz": 1378931,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 840704,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 356352,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1197056,
                            "linear_total": 7077888,
                            "nnz": 1203048,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 759808,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 175104,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 934912,
                            "linear_total": 7077888,
                            "nnz": 940882,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 437248,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 87552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 524800,
                            "linear_total": 7077888,
                            "nnz": 530137,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 13676544,
                    "linear_sparsity": 83.89756944444444,
                    "linear_total": 84934656,
                    "nnz": 37586312,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            1,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            10,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 65.48332050822721
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 78.63765373699148,
                    "f1": 86.33602681050554
                }
            },
            "f1": 85.87492067079658,
            "fill_rate": 0.16102430555555558,
            "speedup": 2.332830121156959
        },
        {
            "annotate": "43",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 81.3434247871334,
                    "f1": 88.42277380990502
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_9_gelupatch2/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test_9_fullpatch2___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test_9_fullpatch2_--34ff303320644b64/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": 1,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": 0,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 2
                },
                "speed": {
                    "cuda_eval_elapsed_time": 25.775183883666994,
                    "eval_elapsed_time": 32.76169525459409
                },
                "speedup": 1.4973469512210646,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1841152,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1996800,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3837952,
                            "linear_total": 7077888,
                            "nnz": 3845972,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 1388544,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2052096,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3440640,
                            "linear_total": 7077888,
                            "nnz": 3448280,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 1069056,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 486912,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1555968,
                            "linear_total": 7077888,
                            "nnz": 1562397,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 940032,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 755712,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1695744,
                            "linear_total": 7077888,
                            "nnz": 1702348,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1741824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2155008,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3896832,
                            "linear_total": 7077888,
                            "nnz": 3904827,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1979392,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2260992,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4240384,
                            "linear_total": 7077888,
                            "nnz": 4248576,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 2034688,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2098176,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4132864,
                            "linear_total": 7077888,
                            "nnz": 4141046,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 2010112,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2021376,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4031488,
                            "linear_total": 7077888,
                            "nnz": 4039620,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1893376,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1669632,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3563008,
                            "linear_total": 7077888,
                            "nnz": 3570847,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 1372160,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1241088,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2613248,
                            "linear_total": 7077888,
                            "nnz": 2620168,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 1839104,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 622080,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2461184,
                            "linear_total": 7077888,
                            "nnz": 2468469,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 1112064,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 374784,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1486848,
                            "linear_total": 7077888,
                            "nnz": 1493076,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 36956160,
                    "linear_sparsity": 56.48871527777778,
                    "linear_total": 84934656,
                    "nnz": 60884348,
                    "pruned_heads": {
                        "0": [],
                        "1": [
                            0,
                            2,
                            3
                        ],
                        "10": [
                            1,
                            4,
                            7
                        ],
                        "11": [
                            8,
                            5
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            4
                        ],
                        "4": [],
                        "5": [],
                        "6": [
                            3
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [],
                        "9": [
                            1,
                            4,
                            5,
                            7
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 44.08800932686459
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch2/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch2/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch2/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 83.15042573320719,
                    "f1": 89.82098982678322
                }
            },
            "f1": 88.42277380990502,
            "fill_rate": 0.4351128472222222,
            "speedup": 1.4973469512210646
        },
        {
            "annotate": "16",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.81456953642385,
                    "f1": 85.84663307531542
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_9_gelupatch3/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test_9_fullpatch3___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test_9_fullpatch3_--37a1c030322f0ff3/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": 1,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": 0,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20
                },
                "speed": {
                    "cuda_eval_elapsed_time": 16.29583140182495,
                    "eval_elapsed_time": 23.167893490754068
                },
                "speedup": 2.368359861715859,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 665600,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 316416,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 982016,
                            "linear_total": 7077888,
                            "nnz": 987662,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 727040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 402432,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1129472,
                            "linear_total": 7077888,
                            "nnz": 1135206,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 456704,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 132096,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 588800,
                            "linear_total": 7077888,
                            "nnz": 594262,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 355328,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 167424,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 522752,
                            "linear_total": 7077888,
                            "nnz": 528205,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1085440,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 532992,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1618432,
                            "linear_total": 7077888,
                            "nnz": 1624795,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1203200,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 625152,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1828352,
                            "linear_total": 7077888,
                            "nnz": 1834839,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1140736,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 579072,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1719808,
                            "linear_total": 7077888,
                            "nnz": 1726361,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 830464,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 536064,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1366528,
                            "linear_total": 7077888,
                            "nnz": 1372573,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 782336,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 446976,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1229312,
                            "linear_total": 7077888,
                            "nnz": 1235395,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 825344,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 327168,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1152512,
                            "linear_total": 7077888,
                            "nnz": 1158453,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 770048,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 175104,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 945152,
                            "linear_total": 7077888,
                            "nnz": 951154,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 433152,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 92160,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 525312,
                            "linear_total": 7077888,
                            "nnz": 530748,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 13608448,
                    "linear_sparsity": 83.97774402006173,
                    "linear_total": 84934656,
                    "nnz": 37518375,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 65.54570916861593
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch3/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch3/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch3/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 78.27814569536424,
                    "f1": 86.128148072438
                }
            },
            "f1": 85.84663307531542,
            "fill_rate": 0.1602225597993826,
            "speedup": 2.368359861715859
        },
        {
            "annotate": "17",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.31315042573321,
                    "f1": 85.64344127195709
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_9_gelupatch3/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test_9_fullpatch3___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test_9_fullpatch3_--37a1c030322f0ff3/checkpoint-85000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": 1,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": 0,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20
                },
                "speed": {
                    "cuda_eval_elapsed_time": 17.414673099517824,
                    "eval_elapsed_time": 24.35595615580678
                },
                "speedup": 2.21619968315338,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 702464,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 350208,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1052672,
                            "linear_total": 7077888,
                            "nnz": 1058404,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 745472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 427008,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1172480,
                            "linear_total": 7077888,
                            "nnz": 1178230,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 479232,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 135168,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 614400,
                            "linear_total": 7077888,
                            "nnz": 619864,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 378880,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 172032,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 550912,
                            "linear_total": 7077888,
                            "nnz": 556432,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1166336,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 552960,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1719296,
                            "linear_total": 7077888,
                            "nnz": 1725768,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1274880,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 640512,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1915392,
                            "linear_total": 7077888,
                            "nnz": 1921985,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1203200,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 603648,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1806848,
                            "linear_total": 7077888,
                            "nnz": 1813481,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 919552,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 559104,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1478656,
                            "linear_total": 7077888,
                            "nnz": 1484812,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 873472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 463872,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1337344,
                            "linear_total": 7077888,
                            "nnz": 1343534,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 916480,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 333312,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1249792,
                            "linear_total": 7077888,
                            "nnz": 1255865,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 866304,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 185856,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1052160,
                            "linear_total": 7077888,
                            "nnz": 1058201,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 492544,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 93696,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 586240,
                            "linear_total": 7077888,
                            "nnz": 591709,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 14536192,
                    "linear_sparsity": 82.88544077932099,
                    "linear_total": 84934656,
                    "nnz": 38447007,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 64.69291751643671
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch3/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch3/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch3/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 78.08893093661305,
                    "f1": 86.2116347721247
                }
            },
            "f1": 85.64344127195709,
            "fill_rate": 0.17114559220679015,
            "speedup": 2.21619968315338
        },
        {
            "annotate": "16",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.67265846736045,
                    "f1": 85.53356219601635
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_9_gelupatch3/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test_9_fullpatch3___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test_9_fullpatch3_--37a1c030322f0ff3/checkpoint-90000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": 1,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": 0,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20
                },
                "speed": {
                    "cuda_eval_elapsed_time": 16.521136096954347,
                    "eval_elapsed_time": 23.466227194294333
                },
                "speedup": 2.3360616835839716,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 676864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 336384,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1013248,
                            "linear_total": 7077888,
                            "nnz": 1018971,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 717824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 419328,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1137152,
                            "linear_total": 7077888,
                            "nnz": 1142897,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 471040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 132096,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 603136,
                            "linear_total": 7077888,
                            "nnz": 608598,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 380928,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 172032,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 552960,
                            "linear_total": 7077888,
                            "nnz": 558448,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1182720,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 546816,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1729536,
                            "linear_total": 7077888,
                            "nnz": 1736004,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1245184,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 632832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1878016,
                            "linear_total": 7077888,
                            "nnz": 1884604,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1203200,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 595968,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1799168,
                            "linear_total": 7077888,
                            "nnz": 1805764,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 870400,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 551424,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1421824,
                            "linear_total": 7077888,
                            "nnz": 1427975,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 873472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 457728,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1331200,
                            "linear_total": 7077888,
                            "nnz": 1337322,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 866304,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 327168,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1193472,
                            "linear_total": 7077888,
                            "nnz": 1199477,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 840704,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 181248,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1021952,
                            "linear_total": 7077888,
                            "nnz": 1027958,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 506880,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 92160,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 599040,
                            "linear_total": 7077888,
                            "nnz": 604476,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 14280704,
                    "linear_sparsity": 83.1862461419753,
                    "linear_total": 84934656,
                    "nnz": 38191216,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 64.92781834852366
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch3/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch3/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_9_fullpatch3/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 78.54304635761589,
                    "f1": 86.1676946355998
                }
            },
            "f1": 85.53356219601635,
            "fill_rate": 0.16813753858024694,
            "speedup": 2.3360616835839716
        }
    ],
    "Block/struct method, bs= 32x32, v=1, s=l": [
        {
            "annotate": "12",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-large-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 1024,
                    "initializer_range": 0.02,
                    "intermediate_size": 4096,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 16,
                    "num_hidden_layers": 24,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 82.33680227057711,
                    "f1": 89.04761607630476
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_large/large_regu_10/checkpoint-215000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 37.50764268493653,
                    "eval_elapsed_time": 44.93039320781827
                },
                "speedup": 1.0289741034797428,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 974848,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 192512,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1167360,
                            "linear_total": 12582912,
                            "nnz": 1174526,
                            "total": 12596224
                        },
                        "1": {
                            "linear_attention_nnz": 306176,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 270336,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 576512,
                            "linear_total": 12582912,
                            "nnz": 583204,
                            "total": 12596224
                        },
                        "10": {
                            "linear_attention_nnz": 1714176,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 995328,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2709504,
                            "linear_total": 12582912,
                            "nnz": 2717926,
                            "total": 12596224
                        },
                        "11": {
                            "linear_attention_nnz": 1875968,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1032192,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2908160,
                            "linear_total": 12582912,
                            "nnz": 2916760,
                            "total": 12596224
                        },
                        "12": {
                            "linear_attention_nnz": 1832960,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1241088,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3074048,
                            "linear_total": 12582912,
                            "nnz": 3082878,
                            "total": 12596224
                        },
                        "13": {
                            "linear_attention_nnz": 2155520,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1179648,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3335168,
                            "linear_total": 12582912,
                            "nnz": 3344128,
                            "total": 12596224
                        },
                        "14": {
                            "linear_attention_nnz": 1942528,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 909312,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2851840,
                            "linear_total": 12582912,
                            "nnz": 2860412,
                            "total": 12596224
                        },
                        "15": {
                            "linear_attention_nnz": 2079744,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 681984,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2761728,
                            "linear_total": 12582912,
                            "nnz": 2770125,
                            "total": 12596224
                        },
                        "16": {
                            "linear_attention_nnz": 1843200,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 473088,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2316288,
                            "linear_total": 12582912,
                            "nnz": 2324615,
                            "total": 12596224
                        },
                        "17": {
                            "linear_attention_nnz": 1582080,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 368640,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1950720,
                            "linear_total": 12582912,
                            "nnz": 1958964,
                            "total": 12596224
                        },
                        "18": {
                            "linear_attention_nnz": 1435648,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 321536,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1757184,
                            "linear_total": 12582912,
                            "nnz": 1765277,
                            "total": 12596224
                        },
                        "19": {
                            "linear_attention_nnz": 717824,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 270336,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 988160,
                            "linear_total": 12582912,
                            "nnz": 995428,
                            "total": 12596224
                        },
                        "2": {
                            "linear_attention_nnz": 297984,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 286720,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 584704,
                            "linear_total": 12582912,
                            "nnz": 591308,
                            "total": 12596224
                        },
                        "20": {
                            "linear_attention_nnz": 334848,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 112640,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 447488,
                            "linear_total": 12582912,
                            "nnz": 454135,
                            "total": 12596224
                        },
                        "21": {
                            "linear_attention_nnz": 358400,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 77824,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 436224,
                            "linear_total": 12582912,
                            "nnz": 443014,
                            "total": 12596224
                        },
                        "22": {
                            "linear_attention_nnz": 134144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 79872,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 214016,
                            "linear_total": 12582912,
                            "nnz": 220231,
                            "total": 12596224
                        },
                        "23": {
                            "linear_attention_nnz": 111616,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 182272,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 293888,
                            "linear_total": 12582912,
                            "nnz": 300185,
                            "total": 12596224
                        },
                        "3": {
                            "linear_attention_nnz": 834560,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 413696,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1248256,
                            "linear_total": 12582912,
                            "nnz": 1255434,
                            "total": 12596224
                        },
                        "4": {
                            "linear_attention_nnz": 381952,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 466944,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 848896,
                            "linear_total": 12582912,
                            "nnz": 855652,
                            "total": 12596224
                        },
                        "5": {
                            "linear_attention_nnz": 406528,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 552960,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 959488,
                            "linear_total": 12582912,
                            "nnz": 966318,
                            "total": 12596224
                        },
                        "6": {
                            "linear_attention_nnz": 522240,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 608256,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1130496,
                            "linear_total": 12582912,
                            "nnz": 1137481,
                            "total": 12596224
                        },
                        "7": {
                            "linear_attention_nnz": 771072,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 438272,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1209344,
                            "linear_total": 12582912,
                            "nnz": 1216534,
                            "total": 12596224
                        },
                        "8": {
                            "linear_attention_nnz": 414720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 661504,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1076224,
                            "linear_total": 12582912,
                            "nnz": 1083267,
                            "total": 12596224
                        },
                        "9": {
                            "linear_attention_nnz": 1091584,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 747520,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1839104,
                            "linear_total": 12582912,
                            "nnz": 1846669,
                            "total": 12596224
                        }
                    },
                    "linear_nnz": 36684800,
                    "linear_sparsity": 87.85230848524306,
                    "linear_total": 301989888,
                    "nnz": 68649433,
                    "pruned_heads": {
                        "0": [
                            2,
                            3,
                            4,
                            7,
                            8,
                            9,
                            10,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            7,
                            8,
                            10,
                            12,
                            13,
                            14
                        ],
                        "11": [
                            0,
                            2,
                            4,
                            5,
                            8,
                            10
                        ],
                        "12": [
                            10,
                            3,
                            13,
                            6
                        ],
                        "13": [
                            2,
                            10,
                            4,
                            12
                        ],
                        "14": [
                            2,
                            3,
                            4,
                            8,
                            11
                        ],
                        "15": [
                            0,
                            5,
                            6,
                            7,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            8,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            11,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            11,
                            13
                        ],
                        "19": [
                            0,
                            2,
                            3,
                            4,
                            9,
                            10,
                            11,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "21": [
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "23": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            13,
                            14
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            12,
                            13,
                            15
                        ]
                    },
                    "total": 334094338,
                    "total_sparsity": 79.45208128609471
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test_large/squad_test_large_regu-10",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test_large/squad_test_large_regu-10",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 8,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test_large/squad_test_large_regu-10",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 89.04761607630476,
            "fill_rate": 0.32393844039351843,
            "speedup": 1.0289741034797428
        },
        {
            "annotate": "12",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-large-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 1024,
                    "initializer_range": 0.02,
                    "intermediate_size": 4096,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 16,
                    "num_hidden_layers": 24,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 82.13812677388836,
                    "f1": 89.03656646065757
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_large/large_regu_10/checkpoint-220000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 37.54432637023926,
                    "eval_elapsed_time": 44.93571184715256
                },
                "speedup": 1.0279687168915141,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 989184,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 192512,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1181696,
                            "linear_total": 12582912,
                            "nnz": 1188862,
                            "total": 12596224
                        },
                        "1": {
                            "linear_attention_nnz": 323584,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 270336,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 593920,
                            "linear_total": 12582912,
                            "nnz": 600612,
                            "total": 12596224
                        },
                        "10": {
                            "linear_attention_nnz": 1745920,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 995328,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2741248,
                            "linear_total": 12582912,
                            "nnz": 2749670,
                            "total": 12596224
                        },
                        "11": {
                            "linear_attention_nnz": 1902592,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1032192,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2934784,
                            "linear_total": 12582912,
                            "nnz": 2943384,
                            "total": 12596224
                        },
                        "12": {
                            "linear_attention_nnz": 1782784,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1241088,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3023872,
                            "linear_total": 12582912,
                            "nnz": 3032670,
                            "total": 12596224
                        },
                        "13": {
                            "linear_attention_nnz": 2147328,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1179648,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3326976,
                            "linear_total": 12582912,
                            "nnz": 3335936,
                            "total": 12596224
                        },
                        "14": {
                            "linear_attention_nnz": 1917952,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 909312,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2827264,
                            "linear_total": 12582912,
                            "nnz": 2835836,
                            "total": 12596224
                        },
                        "15": {
                            "linear_attention_nnz": 2049024,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 681984,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2731008,
                            "linear_total": 12582912,
                            "nnz": 2739405,
                            "total": 12596224
                        },
                        "16": {
                            "linear_attention_nnz": 1820672,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 473088,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2293760,
                            "linear_total": 12582912,
                            "nnz": 2302087,
                            "total": 12596224
                        },
                        "17": {
                            "linear_attention_nnz": 1562624,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 368640,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1931264,
                            "linear_total": 12582912,
                            "nnz": 1939508,
                            "total": 12596224
                        },
                        "18": {
                            "linear_attention_nnz": 1390592,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 321536,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1712128,
                            "linear_total": 12582912,
                            "nnz": 1720221,
                            "total": 12596224
                        },
                        "19": {
                            "linear_attention_nnz": 688128,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 270336,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 958464,
                            "linear_total": 12582912,
                            "nnz": 965700,
                            "total": 12596224
                        },
                        "2": {
                            "linear_attention_nnz": 286720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 286720,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 573440,
                            "linear_total": 12582912,
                            "nnz": 580044,
                            "total": 12596224
                        },
                        "20": {
                            "linear_attention_nnz": 326656,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 112640,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 439296,
                            "linear_total": 12582912,
                            "nnz": 445879,
                            "total": 12596224
                        },
                        "21": {
                            "linear_attention_nnz": 344064,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 77824,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 421888,
                            "linear_total": 12582912,
                            "nnz": 428614,
                            "total": 12596224
                        },
                        "22": {
                            "linear_attention_nnz": 129024,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 79872,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 208896,
                            "linear_total": 12582912,
                            "nnz": 215079,
                            "total": 12596224
                        },
                        "23": {
                            "linear_attention_nnz": 116736,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 182272,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 299008,
                            "linear_total": 12582912,
                            "nnz": 305273,
                            "total": 12596224
                        },
                        "3": {
                            "linear_attention_nnz": 801792,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 413696,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1215488,
                            "linear_total": 12582912,
                            "nnz": 1222666,
                            "total": 12596224
                        },
                        "4": {
                            "linear_attention_nnz": 396288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 466944,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 863232,
                            "linear_total": 12582912,
                            "nnz": 869988,
                            "total": 12596224
                        },
                        "5": {
                            "linear_attention_nnz": 405504,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 552960,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 958464,
                            "linear_total": 12582912,
                            "nnz": 965294,
                            "total": 12596224
                        },
                        "6": {
                            "linear_attention_nnz": 520192,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 604160,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1124352,
                            "linear_total": 12582912,
                            "nnz": 1131335,
                            "total": 12596224
                        },
                        "7": {
                            "linear_attention_nnz": 764928,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 438272,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1203200,
                            "linear_total": 12582912,
                            "nnz": 1210390,
                            "total": 12596224
                        },
                        "8": {
                            "linear_attention_nnz": 423936,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 659456,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1083392,
                            "linear_total": 12582912,
                            "nnz": 1090434,
                            "total": 12596224
                        },
                        "9": {
                            "linear_attention_nnz": 1070080,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 747520,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1817600,
                            "linear_total": 12582912,
                            "nnz": 1825165,
                            "total": 12596224
                        }
                    },
                    "linear_nnz": 36464640,
                    "linear_sparsity": 87.92521158854166,
                    "linear_total": 301989888,
                    "nnz": 68429014,
                    "pruned_heads": {
                        "0": [
                            2,
                            3,
                            4,
                            7,
                            8,
                            9,
                            10,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            7,
                            8,
                            10,
                            12,
                            13,
                            14
                        ],
                        "11": [
                            0,
                            2,
                            4,
                            5,
                            8,
                            10
                        ],
                        "12": [
                            10,
                            3,
                            13,
                            6
                        ],
                        "13": [
                            2,
                            10,
                            4,
                            12
                        ],
                        "14": [
                            2,
                            3,
                            4,
                            8,
                            11
                        ],
                        "15": [
                            0,
                            5,
                            6,
                            7,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            8,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            11,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            11,
                            13
                        ],
                        "19": [
                            0,
                            2,
                            3,
                            4,
                            9,
                            10,
                            11,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "21": [
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "23": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            13,
                            14
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            12,
                            13,
                            15
                        ]
                    },
                    "total": 334094338,
                    "total_sparsity": 79.51805636406804
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test_large/squad_test_large_regu-10",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test_large/squad_test_large_regu-10",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 8,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test_large/squad_test_large_regu-10",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 89.03656646065757,
            "fill_rate": 0.3219943576388893,
            "speedup": 1.0279687168915141
        },
        {
            "annotate": "12",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-large-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 1024,
                    "initializer_range": 0.02,
                    "intermediate_size": 4096,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 16,
                    "num_hidden_layers": 24,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 82.30842005676443,
                    "f1": 89.04987146464723
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_large/large_regu_10/checkpoint-221320",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 37.53598588562012,
                    "eval_elapsed_time": 44.935436787083745
                },
                "speedup": 1.028197131226982,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 978944,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 192512,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1171456,
                            "linear_total": 12582912,
                            "nnz": 1178622,
                            "total": 12596224
                        },
                        "1": {
                            "linear_attention_nnz": 319488,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 270336,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 589824,
                            "linear_total": 12582912,
                            "nnz": 596516,
                            "total": 12596224
                        },
                        "10": {
                            "linear_attention_nnz": 1754112,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 995328,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2749440,
                            "linear_total": 12582912,
                            "nnz": 2757862,
                            "total": 12596224
                        },
                        "11": {
                            "linear_attention_nnz": 1922048,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1032192,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2954240,
                            "linear_total": 12582912,
                            "nnz": 2962840,
                            "total": 12596224
                        },
                        "12": {
                            "linear_attention_nnz": 1775616,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1241088,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3016704,
                            "linear_total": 12582912,
                            "nnz": 3025502,
                            "total": 12596224
                        },
                        "13": {
                            "linear_attention_nnz": 2149376,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1179648,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3329024,
                            "linear_total": 12582912,
                            "nnz": 3337984,
                            "total": 12596224
                        },
                        "14": {
                            "linear_attention_nnz": 1954816,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 909312,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2864128,
                            "linear_total": 12582912,
                            "nnz": 2872700,
                            "total": 12596224
                        },
                        "15": {
                            "linear_attention_nnz": 2065408,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 681984,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2747392,
                            "linear_total": 12582912,
                            "nnz": 2755789,
                            "total": 12596224
                        },
                        "16": {
                            "linear_attention_nnz": 1823744,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 473088,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2296832,
                            "linear_total": 12582912,
                            "nnz": 2305159,
                            "total": 12596224
                        },
                        "17": {
                            "linear_attention_nnz": 1558528,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 368640,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1927168,
                            "linear_total": 12582912,
                            "nnz": 1935412,
                            "total": 12596224
                        },
                        "18": {
                            "linear_attention_nnz": 1356800,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 321536,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1678336,
                            "linear_total": 12582912,
                            "nnz": 1686429,
                            "total": 12596224
                        },
                        "19": {
                            "linear_attention_nnz": 688128,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 270336,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 958464,
                            "linear_total": 12582912,
                            "nnz": 965732,
                            "total": 12596224
                        },
                        "2": {
                            "linear_attention_nnz": 293888,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 286720,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 580608,
                            "linear_total": 12582912,
                            "nnz": 587212,
                            "total": 12596224
                        },
                        "20": {
                            "linear_attention_nnz": 326656,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 112640,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 439296,
                            "linear_total": 12582912,
                            "nnz": 445975,
                            "total": 12596224
                        },
                        "21": {
                            "linear_attention_nnz": 344064,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 77824,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 421888,
                            "linear_total": 12582912,
                            "nnz": 428646,
                            "total": 12596224
                        },
                        "22": {
                            "linear_attention_nnz": 129024,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 79872,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 208896,
                            "linear_total": 12582912,
                            "nnz": 215079,
                            "total": 12596224
                        },
                        "23": {
                            "linear_attention_nnz": 116736,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 182272,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 299008,
                            "linear_total": 12582912,
                            "nnz": 305273,
                            "total": 12596224
                        },
                        "3": {
                            "linear_attention_nnz": 795648,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 413696,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1209344,
                            "linear_total": 12582912,
                            "nnz": 1216522,
                            "total": 12596224
                        },
                        "4": {
                            "linear_attention_nnz": 395264,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 466944,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 862208,
                            "linear_total": 12582912,
                            "nnz": 868964,
                            "total": 12596224
                        },
                        "5": {
                            "linear_attention_nnz": 392192,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 552960,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 945152,
                            "linear_total": 12582912,
                            "nnz": 951982,
                            "total": 12596224
                        },
                        "6": {
                            "linear_attention_nnz": 523264,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 604160,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1127424,
                            "linear_total": 12582912,
                            "nnz": 1134407,
                            "total": 12596224
                        },
                        "7": {
                            "linear_attention_nnz": 784384,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 438272,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1222656,
                            "linear_total": 12582912,
                            "nnz": 1229846,
                            "total": 12596224
                        },
                        "8": {
                            "linear_attention_nnz": 416768,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 659456,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1076224,
                            "linear_total": 12582912,
                            "nnz": 1083266,
                            "total": 12596224
                        },
                        "9": {
                            "linear_attention_nnz": 1069056,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 747520,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1816576,
                            "linear_total": 12582912,
                            "nnz": 1824141,
                            "total": 12596224
                        }
                    },
                    "linear_nnz": 36492288,
                    "linear_sparsity": 87.91605631510416,
                    "linear_total": 301989888,
                    "nnz": 68456822,
                    "pruned_heads": {
                        "0": [
                            2,
                            3,
                            4,
                            7,
                            8,
                            9,
                            10,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            7,
                            8,
                            10,
                            12,
                            13,
                            14
                        ],
                        "11": [
                            0,
                            2,
                            4,
                            5,
                            8,
                            10
                        ],
                        "12": [
                            10,
                            3,
                            13,
                            6
                        ],
                        "13": [
                            2,
                            10,
                            4,
                            12
                        ],
                        "14": [
                            2,
                            3,
                            4,
                            8,
                            11
                        ],
                        "15": [
                            0,
                            5,
                            6,
                            7,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            8,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            11,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            11,
                            13
                        ],
                        "19": [
                            0,
                            2,
                            3,
                            4,
                            9,
                            10,
                            11,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "21": [
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "23": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            13,
                            14
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            12,
                            13,
                            15
                        ]
                    },
                    "total": 334094338,
                    "total_sparsity": 79.50973296650122
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test_large/squad_test_large_regu-10",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test_large/squad_test_large_regu-10",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 8,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test_large/squad_test_large_regu-10",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 89.04987146464723,
            "fill_rate": 0.3222384982638893,
            "speedup": 1.028197131226982
        },
        {
            "annotate": "18",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-large-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 1024,
                    "initializer_range": 0.02,
                    "intermediate_size": 4096,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 16,
                    "num_hidden_layers": 24,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 83.78429517502366,
                    "f1": 90.32458147221426
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_large/large_regu_10_d0.25/checkpoint-210000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 41.496326583862306,
                    "eval_elapsed_time": 49.08256564009935
                },
                "speedup": 0.9300676995438012,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 783360,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 835584,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1618944,
                            "linear_total": 12582912,
                            "nnz": 1626424,
                            "total": 12596224
                        },
                        "1": {
                            "linear_attention_nnz": 326656,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1275904,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1602560,
                            "linear_total": 12582912,
                            "nnz": 1609647,
                            "total": 12596224
                        },
                        "10": {
                            "linear_attention_nnz": 1636352,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2410496,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4046848,
                            "linear_total": 12582912,
                            "nnz": 4056121,
                            "total": 12596224
                        },
                        "11": {
                            "linear_attention_nnz": 1575936,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2510848,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4086784,
                            "linear_total": 12582912,
                            "nnz": 4095818,
                            "total": 12596224
                        },
                        "12": {
                            "linear_attention_nnz": 1203200,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2660352,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3863552,
                            "linear_total": 12582912,
                            "nnz": 3872307,
                            "total": 12596224
                        },
                        "13": {
                            "linear_attention_nnz": 2030592,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2605056,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4635648,
                            "linear_total": 12582912,
                            "nnz": 4645176,
                            "total": 12596224
                        },
                        "14": {
                            "linear_attention_nnz": 1785856,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2299904,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4085760,
                            "linear_total": 12582912,
                            "nnz": 4094851,
                            "total": 12596224
                        },
                        "15": {
                            "linear_attention_nnz": 1946624,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1699840,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3646464,
                            "linear_total": 12582912,
                            "nnz": 3655358,
                            "total": 12596224
                        },
                        "16": {
                            "linear_attention_nnz": 1647616,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1402880,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3050496,
                            "linear_total": 12582912,
                            "nnz": 3059149,
                            "total": 12596224
                        },
                        "17": {
                            "linear_attention_nnz": 1538048,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1097728,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2635776,
                            "linear_total": 12582912,
                            "nnz": 2644472,
                            "total": 12596224
                        },
                        "18": {
                            "linear_attention_nnz": 1169408,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 901120,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2070528,
                            "linear_total": 12582912,
                            "nnz": 2078488,
                            "total": 12596224
                        },
                        "19": {
                            "linear_attention_nnz": 607232,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 739328,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1346560,
                            "linear_total": 12582912,
                            "nnz": 1353929,
                            "total": 12596224
                        },
                        "2": {
                            "linear_attention_nnz": 305152,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1359872,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1665024,
                            "linear_total": 12582912,
                            "nnz": 1672152,
                            "total": 12596224
                        },
                        "20": {
                            "linear_attention_nnz": 396288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 358400,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 754688,
                            "linear_total": 12582912,
                            "nnz": 761551,
                            "total": 12596224
                        },
                        "21": {
                            "linear_attention_nnz": 284672,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 194560,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 479232,
                            "linear_total": 12582912,
                            "nnz": 485695,
                            "total": 12596224
                        },
                        "22": {
                            "linear_attention_nnz": 70656,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 180224,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 250880,
                            "linear_total": 12582912,
                            "nnz": 256728,
                            "total": 12596224
                        },
                        "23": {
                            "linear_attention_nnz": 111616,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 323584,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 435200,
                            "linear_total": 12582912,
                            "nnz": 441598,
                            "total": 12596224
                        },
                        "3": {
                            "linear_attention_nnz": 626688,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1685504,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2312192,
                            "linear_total": 12582912,
                            "nnz": 2319831,
                            "total": 12596224
                        },
                        "4": {
                            "linear_attention_nnz": 369664,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1767424,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2137088,
                            "linear_total": 12582912,
                            "nnz": 2144479,
                            "total": 12596224
                        },
                        "5": {
                            "linear_attention_nnz": 463872,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1873920,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2337792,
                            "linear_total": 12582912,
                            "nnz": 2345331,
                            "total": 12596224
                        },
                        "6": {
                            "linear_attention_nnz": 294912,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2054144,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2349056,
                            "linear_total": 12582912,
                            "nnz": 2356491,
                            "total": 12596224
                        },
                        "7": {
                            "linear_attention_nnz": 613376,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1773568,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2386944,
                            "linear_total": 12582912,
                            "nnz": 2394690,
                            "total": 12596224
                        },
                        "8": {
                            "linear_attention_nnz": 208896,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1968128,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2177024,
                            "linear_total": 12582912,
                            "nnz": 2184321,
                            "total": 12596224
                        },
                        "9": {
                            "linear_attention_nnz": 923648,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1986560,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2910208,
                            "linear_total": 12582912,
                            "nnz": 2918282,
                            "total": 12596224
                        }
                    },
                    "linear_nnz": 56885248,
                    "linear_sparsity": 81.16319444444444,
                    "linear_total": 301989888,
                    "nnz": 88857851,
                    "pruned_heads": {
                        "0": [
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            7,
                            8,
                            10,
                            12,
                            13
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            8,
                            10
                        ],
                        "12": [
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            13
                        ],
                        "13": [
                            10,
                            2,
                            3,
                            12
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            11
                        ],
                        "15": [
                            0,
                            5,
                            6,
                            7,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            8,
                            10,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            11,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            11,
                            12,
                            13
                        ],
                        "19": [
                            0,
                            2,
                            3,
                            4,
                            9,
                            10,
                            11,
                            13,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "21": [
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "23": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            12,
                            13,
                            15
                        ]
                    },
                    "total": 334094338,
                    "total_sparsity": 73.40336518962498
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 10000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test_large/squad_test_large_regu_10_d0.25",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test_large/squad_test_large_regu_10_d0.25",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 8,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test_large/squad_test_large_regu_10_d0.25",
                    "save_steps": 10000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 90.32458147221426,
            "fill_rate": 0.5023148148148149,
            "speedup": 0.9300676995438012
        },
        {
            "annotate": "18",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-large-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 1024,
                    "initializer_range": 0.02,
                    "intermediate_size": 4096,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 16,
                    "num_hidden_layers": 24,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 83.66130558183538,
                    "f1": 90.22195941338013
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_large/large_regu_10_d0.25/checkpoint-221320",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 41.275371505737304,
                    "eval_elapsed_time": 48.98561626393348
                },
                "speedup": 0.9350465325310627,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 766976,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 831488,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1598464,
                            "linear_total": 12582912,
                            "nnz": 1605910,
                            "total": 12596224
                        },
                        "1": {
                            "linear_attention_nnz": 338944,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1273856,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1612800,
                            "linear_total": 12582912,
                            "nnz": 1619886,
                            "total": 12596224
                        },
                        "10": {
                            "linear_attention_nnz": 1596416,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2408448,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4004864,
                            "linear_total": 12582912,
                            "nnz": 4014136,
                            "total": 12596224
                        },
                        "11": {
                            "linear_attention_nnz": 1615872,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2508800,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4124672,
                            "linear_total": 12582912,
                            "nnz": 4133705,
                            "total": 12596224
                        },
                        "12": {
                            "linear_attention_nnz": 1205248,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2658304,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3863552,
                            "linear_total": 12582912,
                            "nnz": 3872306,
                            "total": 12596224
                        },
                        "13": {
                            "linear_attention_nnz": 2006016,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2603008,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4609024,
                            "linear_total": 12582912,
                            "nnz": 4618551,
                            "total": 12596224
                        },
                        "14": {
                            "linear_attention_nnz": 1718272,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2299904,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4018176,
                            "linear_total": 12582912,
                            "nnz": 4027267,
                            "total": 12596224
                        },
                        "15": {
                            "linear_attention_nnz": 1935360,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1699840,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3635200,
                            "linear_total": 12582912,
                            "nnz": 3644094,
                            "total": 12596224
                        },
                        "16": {
                            "linear_attention_nnz": 1612800,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1402880,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3015680,
                            "linear_total": 12582912,
                            "nnz": 3024333,
                            "total": 12596224
                        },
                        "17": {
                            "linear_attention_nnz": 1502208,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1097728,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2599936,
                            "linear_total": 12582912,
                            "nnz": 2608632,
                            "total": 12596224
                        },
                        "18": {
                            "linear_attention_nnz": 1167360,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 901120,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2068480,
                            "linear_total": 12582912,
                            "nnz": 2076440,
                            "total": 12596224
                        },
                        "19": {
                            "linear_attention_nnz": 601088,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 739328,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1340416,
                            "linear_total": 12582912,
                            "nnz": 1347785,
                            "total": 12596224
                        },
                        "2": {
                            "linear_attention_nnz": 305152,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1357824,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1662976,
                            "linear_total": 12582912,
                            "nnz": 1670103,
                            "total": 12596224
                        },
                        "20": {
                            "linear_attention_nnz": 364544,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 356352,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 720896,
                            "linear_total": 12582912,
                            "nnz": 727758,
                            "total": 12596224
                        },
                        "21": {
                            "linear_attention_nnz": 274432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 194560,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 468992,
                            "linear_total": 12582912,
                            "nnz": 475519,
                            "total": 12596224
                        },
                        "22": {
                            "linear_attention_nnz": 70656,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 180224,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 250880,
                            "linear_total": 12582912,
                            "nnz": 256728,
                            "total": 12596224
                        },
                        "23": {
                            "linear_attention_nnz": 102400,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 321536,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 423936,
                            "linear_total": 12582912,
                            "nnz": 430301,
                            "total": 12596224
                        },
                        "3": {
                            "linear_attention_nnz": 621568,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1685504,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2307072,
                            "linear_total": 12582912,
                            "nnz": 2314711,
                            "total": 12596224
                        },
                        "4": {
                            "linear_attention_nnz": 377856,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1767424,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2145280,
                            "linear_total": 12582912,
                            "nnz": 2152671,
                            "total": 12596224
                        },
                        "5": {
                            "linear_attention_nnz": 460800,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1871872,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2332672,
                            "linear_total": 12582912,
                            "nnz": 2340210,
                            "total": 12596224
                        },
                        "6": {
                            "linear_attention_nnz": 309248,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2054144,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2363392,
                            "linear_total": 12582912,
                            "nnz": 2370827,
                            "total": 12596224
                        },
                        "7": {
                            "linear_attention_nnz": 583680,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1773568,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2357248,
                            "linear_total": 12582912,
                            "nnz": 2364994,
                            "total": 12596224
                        },
                        "8": {
                            "linear_attention_nnz": 215040,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1966080,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2181120,
                            "linear_total": 12582912,
                            "nnz": 2188448,
                            "total": 12596224
                        },
                        "9": {
                            "linear_attention_nnz": 916480,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1986560,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2903040,
                            "linear_total": 12582912,
                            "nnz": 2911082,
                            "total": 12596224
                        }
                    },
                    "linear_nnz": 56608768,
                    "linear_sparsity": 81.25474717881944,
                    "linear_total": 301989888,
                    "nnz": 88581359,
                    "pruned_heads": {
                        "0": [
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            7,
                            8,
                            10,
                            12,
                            13
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            8,
                            10
                        ],
                        "12": [
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            13
                        ],
                        "13": [
                            10,
                            2,
                            3,
                            12
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            11
                        ],
                        "15": [
                            0,
                            5,
                            6,
                            7,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            8,
                            10,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            11,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            11,
                            12,
                            13
                        ],
                        "19": [
                            0,
                            2,
                            3,
                            4,
                            9,
                            10,
                            11,
                            13,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "21": [
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "23": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            12,
                            13,
                            15
                        ]
                    },
                    "total": 334094338,
                    "total_sparsity": 73.4861238504437
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 10000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test_large/squad_test_large_regu_10_d0.25",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test_large/squad_test_large_regu_10_d0.25",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 8,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test_large/squad_test_large_regu_10_d0.25",
                    "save_steps": 10000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 90.22195941338013,
            "fill_rate": 0.4998734085648149,
            "speedup": 0.9350465325310627
        },
        {
            "annotate": "5",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-large-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 1024,
                    "initializer_range": 0.02,
                    "intermediate_size": 4096,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 16,
                    "num_hidden_layers": 24,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.92147587511826,
                    "f1": 86.66302391758462
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_large/large_regu_40/checkpoint-221320",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 40
                },
                "speed": {
                    "cuda_eval_elapsed_time": 28.611265159606933,
                    "eval_elapsed_time": 36.00721236690879
                },
                "speedup": 1.3489229780673324,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 668672,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 88064,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 756736,
                            "linear_total": 12582912,
                            "nnz": 763595,
                            "total": 12596224
                        },
                        "1": {
                            "linear_attention_nnz": 232448,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 102400,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 334848,
                            "linear_total": 12582912,
                            "nnz": 341330,
                            "total": 12596224
                        },
                        "10": {
                            "linear_attention_nnz": 864256,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 442368,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1306624,
                            "linear_total": 12582912,
                            "nnz": 1314200,
                            "total": 12596224
                        },
                        "11": {
                            "linear_attention_nnz": 985088,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 462848,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1447936,
                            "linear_total": 12582912,
                            "nnz": 1455586,
                            "total": 12596224
                        },
                        "12": {
                            "linear_attention_nnz": 726016,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 557056,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1283072,
                            "linear_total": 12582912,
                            "nnz": 1290384,
                            "total": 12596224
                        },
                        "13": {
                            "linear_attention_nnz": 1306624,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 507904,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1814528,
                            "linear_total": 12582912,
                            "nnz": 1822616,
                            "total": 12596224
                        },
                        "14": {
                            "linear_attention_nnz": 1107968,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 362496,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1470464,
                            "linear_total": 12582912,
                            "nnz": 1478321,
                            "total": 12596224
                        },
                        "15": {
                            "linear_attention_nnz": 1074176,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 278528,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1352704,
                            "linear_total": 12582912,
                            "nnz": 1360360,
                            "total": 12596224
                        },
                        "16": {
                            "linear_attention_nnz": 951296,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 188416,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1139712,
                            "linear_total": 12582912,
                            "nnz": 1147196,
                            "total": 12596224
                        },
                        "17": {
                            "linear_attention_nnz": 795648,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 188416,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 984064,
                            "linear_total": 12582912,
                            "nnz": 991868,
                            "total": 12596224
                        },
                        "18": {
                            "linear_attention_nnz": 706560,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 141312,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 847872,
                            "linear_total": 12582912,
                            "nnz": 855333,
                            "total": 12596224
                        },
                        "19": {
                            "linear_attention_nnz": 290816,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 137216,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 428032,
                            "linear_total": 12582912,
                            "nnz": 434563,
                            "total": 12596224
                        },
                        "2": {
                            "linear_attention_nnz": 146432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 90112,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 236544,
                            "linear_total": 12582912,
                            "nnz": 242732,
                            "total": 12596224
                        },
                        "20": {
                            "linear_attention_nnz": 186368,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 57344,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 243712,
                            "linear_total": 12582912,
                            "nnz": 249916,
                            "total": 12596224
                        },
                        "21": {
                            "linear_attention_nnz": 194560,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 40960,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 235520,
                            "linear_total": 12582912,
                            "nnz": 241748,
                            "total": 12596224
                        },
                        "22": {
                            "linear_attention_nnz": 46080,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 40960,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 87040,
                            "linear_total": 12582912,
                            "nnz": 92724,
                            "total": 12596224
                        },
                        "23": {
                            "linear_attention_nnz": 54272,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 102400,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 156672,
                            "linear_total": 12582912,
                            "nnz": 162450,
                            "total": 12596224
                        },
                        "3": {
                            "linear_attention_nnz": 359424,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 155648,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 515072,
                            "linear_total": 12582912,
                            "nnz": 521644,
                            "total": 12596224
                        },
                        "4": {
                            "linear_attention_nnz": 349184,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 143360,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 492544,
                            "linear_total": 12582912,
                            "nnz": 499142,
                            "total": 12596224
                        },
                        "5": {
                            "linear_attention_nnz": 270336,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 167936,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 438272,
                            "linear_total": 12582912,
                            "nnz": 444850,
                            "total": 12596224
                        },
                        "6": {
                            "linear_attention_nnz": 196608,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 212992,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 409600,
                            "linear_total": 12582912,
                            "nnz": 415976,
                            "total": 12596224
                        },
                        "7": {
                            "linear_attention_nnz": 494592,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 178176,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 672768,
                            "linear_total": 12582912,
                            "nnz": 679575,
                            "total": 12596224
                        },
                        "8": {
                            "linear_attention_nnz": 173056,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 229376,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 402432,
                            "linear_total": 12582912,
                            "nnz": 408880,
                            "total": 12596224
                        },
                        "9": {
                            "linear_attention_nnz": 630784,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 370688,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1001472,
                            "linear_total": 12582912,
                            "nnz": 1008469,
                            "total": 12596224
                        }
                    },
                    "linear_nnz": 18058240,
                    "linear_sparsity": 94.02025010850694,
                    "linear_total": 301989888,
                    "nnz": 50008420,
                    "pruned_heads": {
                        "0": [
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            14
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            12
                        ],
                        "12": [
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            14
                        ],
                        "13": [
                            2,
                            3,
                            4,
                            10,
                            11,
                            12
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            11
                        ],
                        "15": [
                            0,
                            2,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            8,
                            10,
                            12,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            11,
                            12,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            11,
                            12,
                            13
                        ],
                        "19": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "21": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "23": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            12,
                            13,
                            15
                        ]
                    },
                    "total": 334094338,
                    "total_sparsity": 85.0316469595483
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 10000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test_large/squad_test_large_regu_40",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test_large/squad_test_large_regu_40",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 8,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test_large/squad_test_large_regu_40",
                    "save_steps": 10000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.66302391758462,
            "fill_rate": 0.15945999710648154,
            "speedup": 1.3489229780673324
        },
        {
            "annotate": "8",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-large-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 1024,
                    "initializer_range": 0.02,
                    "intermediate_size": 4096,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 16,
                    "num_hidden_layers": 24,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.59602649006622,
                    "f1": 87.8561484925226
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_large/large_regu_40_d0.25/checkpoint-220000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 40
                },
                "speed": {
                    "cuda_eval_elapsed_time": 29.83378296661377,
                    "eval_elapsed_time": 37.31617963500321
                },
                "speedup": 1.2936473074353696,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 340992,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 253952,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 594944,
                            "linear_total": 12582912,
                            "nnz": 601564,
                            "total": 12596224
                        },
                        "1": {
                            "linear_attention_nnz": 141312,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 432128,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 573440,
                            "linear_total": 12582912,
                            "nnz": 579955,
                            "total": 12596224
                        },
                        "10": {
                            "linear_attention_nnz": 832512,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1210368,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2042880,
                            "linear_total": 12582912,
                            "nnz": 2050735,
                            "total": 12596224
                        },
                        "11": {
                            "linear_attention_nnz": 765952,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1277952,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2043904,
                            "linear_total": 12582912,
                            "nnz": 2051536,
                            "total": 12596224
                        },
                        "12": {
                            "linear_attention_nnz": 720896,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1400832,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2121728,
                            "linear_total": 12582912,
                            "nnz": 2129420,
                            "total": 12596224
                        },
                        "13": {
                            "linear_attention_nnz": 1234944,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1464320,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2699264,
                            "linear_total": 12582912,
                            "nnz": 2707787,
                            "total": 12596224
                        },
                        "14": {
                            "linear_attention_nnz": 879616,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1122304,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2001920,
                            "linear_total": 12582912,
                            "nnz": 2009956,
                            "total": 12596224
                        },
                        "15": {
                            "linear_attention_nnz": 917504,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 778240,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1695744,
                            "linear_total": 12582912,
                            "nnz": 1703324,
                            "total": 12596224
                        },
                        "16": {
                            "linear_attention_nnz": 793600,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 532480,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1326080,
                            "linear_total": 12582912,
                            "nnz": 1333572,
                            "total": 12596224
                        },
                        "17": {
                            "linear_attention_nnz": 726016,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 456704,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1182720,
                            "linear_total": 12582912,
                            "nnz": 1190495,
                            "total": 12596224
                        },
                        "18": {
                            "linear_attention_nnz": 656384,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 440320,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1096704,
                            "linear_total": 12582912,
                            "nnz": 1104087,
                            "total": 12596224
                        },
                        "19": {
                            "linear_attention_nnz": 281600,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 362496,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 644096,
                            "linear_total": 12582912,
                            "nnz": 650801,
                            "total": 12596224
                        },
                        "2": {
                            "linear_attention_nnz": 238592,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 450560,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 689152,
                            "linear_total": 12582912,
                            "nnz": 695836,
                            "total": 12596224
                        },
                        "20": {
                            "linear_attention_nnz": 137216,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 184320,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 321536,
                            "linear_total": 12582912,
                            "nnz": 327738,
                            "total": 12596224
                        },
                        "21": {
                            "linear_attention_nnz": 175104,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 112640,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 287744,
                            "linear_total": 12582912,
                            "nnz": 293879,
                            "total": 12596224
                        },
                        "22": {
                            "linear_attention_nnz": 54272,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 114688,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 168960,
                            "linear_total": 12582912,
                            "nnz": 174872,
                            "total": 12596224
                        },
                        "23": {
                            "linear_attention_nnz": 24576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 184320,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 208896,
                            "linear_total": 12582912,
                            "nnz": 214458,
                            "total": 12596224
                        },
                        "3": {
                            "linear_attention_nnz": 320512,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 548864,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 869376,
                            "linear_total": 12582912,
                            "nnz": 876172,
                            "total": 12596224
                        },
                        "4": {
                            "linear_attention_nnz": 332800,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 614400,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 947200,
                            "linear_total": 12582912,
                            "nnz": 954028,
                            "total": 12596224
                        },
                        "5": {
                            "linear_attention_nnz": 147456,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 839680,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 987136,
                            "linear_total": 12582912,
                            "nnz": 993786,
                            "total": 12596224
                        },
                        "6": {
                            "linear_attention_nnz": 166912,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 858112,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1025024,
                            "linear_total": 12582912,
                            "nnz": 1031555,
                            "total": 12596224
                        },
                        "7": {
                            "linear_attention_nnz": 376832,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 636928,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1013760,
                            "linear_total": 12582912,
                            "nnz": 1020759,
                            "total": 12596224
                        },
                        "8": {
                            "linear_attention_nnz": 145408,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 847872,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 993280,
                            "linear_total": 12582912,
                            "nnz": 999934,
                            "total": 12596224
                        },
                        "9": {
                            "linear_attention_nnz": 466944,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 901120,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1368064,
                            "linear_total": 12582912,
                            "nnz": 1375160,
                            "total": 12596224
                        }
                    },
                    "linear_nnz": 26903552,
                    "linear_sparsity": 91.09124077690971,
                    "linear_total": 301989888,
                    "nnz": 58856371,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            14
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            12,
                            15
                        ],
                        "12": [
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "13": [
                            2,
                            3,
                            4,
                            10,
                            11,
                            12
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            11,
                            13
                        ],
                        "15": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            11,
                            12,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            11,
                            12,
                            13
                        ],
                        "19": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            9,
                            10,
                            11,
                            13,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "21": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "23": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12,
                            13,
                            15
                        ]
                    },
                    "total": 334094338,
                    "total_sparsity": 82.38330785480117
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 10000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test_large/squad_test_large_regu_40_d0.25",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test_large/squad_test_large_regu_40_d0.25",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 8,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test_large/squad_test_large_regu_40_d0.25",
                    "save_steps": 10000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.8561484925226,
            "fill_rate": 0.23756691261574106,
            "speedup": 1.2936473074353696
        },
        {
            "annotate": "8",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-large-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 1024,
                    "initializer_range": 0.02,
                    "intermediate_size": 4096,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 16,
                    "num_hidden_layers": 24,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.4635761589404,
                    "f1": 87.71992570037945
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_large/large_regu_40_d0.25/checkpoint-221320",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 40
                },
                "speed": {
                    "cuda_eval_elapsed_time": 29.83577773284912,
                    "eval_elapsed_time": 37.33651804598048
                },
                "speedup": 1.293560816511874,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 365568,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 253952,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 619520,
                            "linear_total": 12582912,
                            "nnz": 626172,
                            "total": 12596224
                        },
                        "1": {
                            "linear_attention_nnz": 137216,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 432128,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 569344,
                            "linear_total": 12582912,
                            "nnz": 575859,
                            "total": 12596224
                        },
                        "10": {
                            "linear_attention_nnz": 826368,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1210368,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2036736,
                            "linear_total": 12582912,
                            "nnz": 2044591,
                            "total": 12596224
                        },
                        "11": {
                            "linear_attention_nnz": 764928,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1277952,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2042880,
                            "linear_total": 12582912,
                            "nnz": 2050512,
                            "total": 12596224
                        },
                        "12": {
                            "linear_attention_nnz": 737280,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1400832,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2138112,
                            "linear_total": 12582912,
                            "nnz": 2145804,
                            "total": 12596224
                        },
                        "13": {
                            "linear_attention_nnz": 1224704,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1464320,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2689024,
                            "linear_total": 12582912,
                            "nnz": 2697547,
                            "total": 12596224
                        },
                        "14": {
                            "linear_attention_nnz": 869376,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1122304,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1991680,
                            "linear_total": 12582912,
                            "nnz": 1999716,
                            "total": 12596224
                        },
                        "15": {
                            "linear_attention_nnz": 924672,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 778240,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1702912,
                            "linear_total": 12582912,
                            "nnz": 1710492,
                            "total": 12596224
                        },
                        "16": {
                            "linear_attention_nnz": 782336,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 532480,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1314816,
                            "linear_total": 12582912,
                            "nnz": 1322308,
                            "total": 12596224
                        },
                        "17": {
                            "linear_attention_nnz": 720896,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 456704,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1177600,
                            "linear_total": 12582912,
                            "nnz": 1185343,
                            "total": 12596224
                        },
                        "18": {
                            "linear_attention_nnz": 655360,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 440320,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1095680,
                            "linear_total": 12582912,
                            "nnz": 1103159,
                            "total": 12596224
                        },
                        "19": {
                            "linear_attention_nnz": 288768,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 362496,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 651264,
                            "linear_total": 12582912,
                            "nnz": 658001,
                            "total": 12596224
                        },
                        "2": {
                            "linear_attention_nnz": 242688,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 450560,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 693248,
                            "linear_total": 12582912,
                            "nnz": 699900,
                            "total": 12596224
                        },
                        "20": {
                            "linear_attention_nnz": 141312,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 184320,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 325632,
                            "linear_total": 12582912,
                            "nnz": 331802,
                            "total": 12596224
                        },
                        "21": {
                            "linear_attention_nnz": 173056,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 112640,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 285696,
                            "linear_total": 12582912,
                            "nnz": 291863,
                            "total": 12596224
                        },
                        "22": {
                            "linear_attention_nnz": 55296,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 114688,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 169984,
                            "linear_total": 12582912,
                            "nnz": 175928,
                            "total": 12596224
                        },
                        "23": {
                            "linear_attention_nnz": 24576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 184320,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 208896,
                            "linear_total": 12582912,
                            "nnz": 214458,
                            "total": 12596224
                        },
                        "3": {
                            "linear_attention_nnz": 327680,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 548864,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 876544,
                            "linear_total": 12582912,
                            "nnz": 883340,
                            "total": 12596224
                        },
                        "4": {
                            "linear_attention_nnz": 332800,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 614400,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 947200,
                            "linear_total": 12582912,
                            "nnz": 954028,
                            "total": 12596224
                        },
                        "5": {
                            "linear_attention_nnz": 139264,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 839680,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 978944,
                            "linear_total": 12582912,
                            "nnz": 985594,
                            "total": 12596224
                        },
                        "6": {
                            "linear_attention_nnz": 165888,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 858112,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1024000,
                            "linear_total": 12582912,
                            "nnz": 1030595,
                            "total": 12596224
                        },
                        "7": {
                            "linear_attention_nnz": 381952,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 636928,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1018880,
                            "linear_total": 12582912,
                            "nnz": 1025879,
                            "total": 12596224
                        },
                        "8": {
                            "linear_attention_nnz": 134144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 847872,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 982016,
                            "linear_total": 12582912,
                            "nnz": 988670,
                            "total": 12596224
                        },
                        "9": {
                            "linear_attention_nnz": 489472,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 901120,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1390592,
                            "linear_total": 12582912,
                            "nnz": 1397688,
                            "total": 12596224
                        }
                    },
                    "linear_nnz": 26931200,
                    "linear_sparsity": 91.08208550347221,
                    "linear_total": 301989888,
                    "nnz": 58884211,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            14
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            12,
                            15
                        ],
                        "12": [
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "13": [
                            2,
                            3,
                            4,
                            10,
                            11,
                            12
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            11,
                            13
                        ],
                        "15": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            11,
                            12,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            11,
                            12,
                            13
                        ],
                        "19": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            9,
                            10,
                            11,
                            13,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "21": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "23": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12,
                            13,
                            15
                        ]
                    },
                    "total": 334094338,
                    "total_sparsity": 82.37497487910136
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 10000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test_large/squad_test_large_regu_40_d0.25",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test_large/squad_test_large_regu_40_d0.25",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 8,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test_large/squad_test_large_regu_40_d0.25",
                    "save_steps": 10000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.71992570037945,
            "fill_rate": 0.23781105324074106,
            "speedup": 1.293560816511874
        },
        {
            "annotate": "4",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-large-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 1024,
                    "initializer_range": 0.02,
                    "intermediate_size": 4096,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 16,
                    "num_hidden_layers": 24,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.27814569536424,
                    "f1": 86.23578242662717
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_large/large_regu_60/checkpoint-221320",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 60
                },
                "speed": {
                    "cuda_eval_elapsed_time": 26.930138206481935,
                    "eval_elapsed_time": 34.07285923091695
                },
                "speedup": 1.4331301499255447,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 527360,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 71680,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 599040,
                            "linear_total": 12582912,
                            "nnz": 605731,
                            "total": 12596224
                        },
                        "1": {
                            "linear_attention_nnz": 167936,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 59392,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 227328,
                            "linear_total": 12582912,
                            "nnz": 233661,
                            "total": 12596224
                        },
                        "10": {
                            "linear_attention_nnz": 719872,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 360448,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1080320,
                            "linear_total": 12582912,
                            "nnz": 1087568,
                            "total": 12596224
                        },
                        "11": {
                            "linear_attention_nnz": 775168,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 352256,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1127424,
                            "linear_total": 12582912,
                            "nnz": 1134732,
                            "total": 12596224
                        },
                        "12": {
                            "linear_attention_nnz": 614400,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 460800,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1075200,
                            "linear_total": 12582912,
                            "nnz": 1082273,
                            "total": 12596224
                        },
                        "13": {
                            "linear_attention_nnz": 1112064,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 382976,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1495040,
                            "linear_total": 12582912,
                            "nnz": 1502907,
                            "total": 12596224
                        },
                        "14": {
                            "linear_attention_nnz": 849920,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 276480,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1126400,
                            "linear_total": 12582912,
                            "nnz": 1134023,
                            "total": 12596224
                        },
                        "15": {
                            "linear_attention_nnz": 825344,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 258048,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1083392,
                            "linear_total": 12582912,
                            "nnz": 1090814,
                            "total": 12596224
                        },
                        "16": {
                            "linear_attention_nnz": 813056,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 147456,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 960512,
                            "linear_total": 12582912,
                            "nnz": 967816,
                            "total": 12596224
                        },
                        "17": {
                            "linear_attention_nnz": 618496,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 145408,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 763904,
                            "linear_total": 12582912,
                            "nnz": 771239,
                            "total": 12596224
                        },
                        "18": {
                            "linear_attention_nnz": 572416,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 116736,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 689152,
                            "linear_total": 12582912,
                            "nnz": 696409,
                            "total": 12596224
                        },
                        "19": {
                            "linear_attention_nnz": 216064,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 100352,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 316416,
                            "linear_total": 12582912,
                            "nnz": 322737,
                            "total": 12596224
                        },
                        "2": {
                            "linear_attention_nnz": 84992,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 63488,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 148480,
                            "linear_total": 12582912,
                            "nnz": 154687,
                            "total": 12596224
                        },
                        "20": {
                            "linear_attention_nnz": 165888,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 49152,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 215040,
                            "linear_total": 12582912,
                            "nnz": 221240,
                            "total": 12596224
                        },
                        "21": {
                            "linear_attention_nnz": 153600,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 40960,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 194560,
                            "linear_total": 12582912,
                            "nnz": 200660,
                            "total": 12596224
                        },
                        "22": {
                            "linear_attention_nnz": 53248,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 49152,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 102400,
                            "linear_total": 12582912,
                            "nnz": 108184,
                            "total": 12596224
                        },
                        "23": {
                            "linear_attention_nnz": 26624,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 81920,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 108544,
                            "linear_total": 12582912,
                            "nnz": 114056,
                            "total": 12596224
                        },
                        "3": {
                            "linear_attention_nnz": 355328,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 100352,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 455680,
                            "linear_total": 12582912,
                            "nnz": 462257,
                            "total": 12596224
                        },
                        "4": {
                            "linear_attention_nnz": 350208,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 133120,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 483328,
                            "linear_total": 12582912,
                            "nnz": 489921,
                            "total": 12596224
                        },
                        "5": {
                            "linear_attention_nnz": 24576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 129024,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 153600,
                            "linear_total": 12582912,
                            "nnz": 159135,
                            "total": 12596224
                        },
                        "6": {
                            "linear_attention_nnz": 187392,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 155648,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 343040,
                            "linear_total": 12582912,
                            "nnz": 349292,
                            "total": 12596224
                        },
                        "7": {
                            "linear_attention_nnz": 429056,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 145408,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 574464,
                            "linear_total": 12582912,
                            "nnz": 581223,
                            "total": 12596224
                        },
                        "8": {
                            "linear_attention_nnz": 129024,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 174080,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 303104,
                            "linear_total": 12582912,
                            "nnz": 309301,
                            "total": 12596224
                        },
                        "9": {
                            "linear_attention_nnz": 491520,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 274432,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 765952,
                            "linear_total": 12582912,
                            "nnz": 772742,
                            "total": 12596224
                        }
                    },
                    "linear_nnz": 14392320,
                    "linear_sparsity": 95.23417154947916,
                    "linear_total": 301989888,
                    "nnz": 46337570,
                    "pruned_heads": {
                        "0": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            3,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            12,
                            15
                        ],
                        "12": [
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "13": [
                            2,
                            3,
                            4,
                            9,
                            10,
                            11,
                            12
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            11,
                            12
                        ],
                        "15": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            8,
                            10,
                            12,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            8,
                            11,
                            12,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            11,
                            13
                        ],
                        "19": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "2": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "21": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "23": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12,
                            13,
                            15
                        ]
                    },
                    "total": 334094338,
                    "total_sparsity": 86.13039350580075
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 10000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test_large/large_regu_60",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test_large/large_regu_60",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 8,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test_large/large_regu_60",
                    "save_steps": 10000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.23578242662717,
            "fill_rate": 0.12708875868055594,
            "speedup": 1.4331301499255447
        },
        {
            "annotate": "7",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-large-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 1024,
                    "initializer_range": 0.02,
                    "intermediate_size": 4096,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 16,
                    "num_hidden_layers": 24,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.50804162724693,
                    "f1": 86.9851273164745
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_large/large_regu_60_d0.25/checkpoint-221320",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 60
                },
                "speed": {
                    "cuda_eval_elapsed_time": 28.35688896179199,
                    "eval_elapsed_time": 35.278680617921054
                },
                "speedup": 1.3610235261481995,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 388096,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 231424,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 619520,
                            "linear_total": 12582912,
                            "nnz": 626257,
                            "total": 12596224
                        },
                        "1": {
                            "linear_attention_nnz": 78848,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 290816,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 369664,
                            "linear_total": 12582912,
                            "nnz": 375982,
                            "total": 12596224
                        },
                        "10": {
                            "linear_attention_nnz": 696320,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 923648,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1619968,
                            "linear_total": 12582912,
                            "nnz": 1627427,
                            "total": 12596224
                        },
                        "11": {
                            "linear_attention_nnz": 627712,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1075200,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1702912,
                            "linear_total": 12582912,
                            "nnz": 1710349,
                            "total": 12596224
                        },
                        "12": {
                            "linear_attention_nnz": 573440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1126400,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1699840,
                            "linear_total": 12582912,
                            "nnz": 1707206,
                            "total": 12596224
                        },
                        "13": {
                            "linear_attention_nnz": 932864,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1179648,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2112512,
                            "linear_total": 12582912,
                            "nnz": 2120576,
                            "total": 12596224
                        },
                        "14": {
                            "linear_attention_nnz": 676864,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 929792,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1606656,
                            "linear_total": 12582912,
                            "nnz": 1614374,
                            "total": 12596224
                        },
                        "15": {
                            "linear_attention_nnz": 807936,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 622592,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1430528,
                            "linear_total": 12582912,
                            "nnz": 1438000,
                            "total": 12596224
                        },
                        "16": {
                            "linear_attention_nnz": 628736,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 403456,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1032192,
                            "linear_total": 12582912,
                            "nnz": 1039525,
                            "total": 12596224
                        },
                        "17": {
                            "linear_attention_nnz": 567296,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 360448,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 927744,
                            "linear_total": 12582912,
                            "nnz": 935024,
                            "total": 12596224
                        },
                        "18": {
                            "linear_attention_nnz": 482304,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 335872,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 818176,
                            "linear_total": 12582912,
                            "nnz": 825348,
                            "total": 12596224
                        },
                        "19": {
                            "linear_attention_nnz": 198656,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 307200,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 505856,
                            "linear_total": 12582912,
                            "nnz": 512118,
                            "total": 12596224
                        },
                        "2": {
                            "linear_attention_nnz": 220160,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 331776,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 551936,
                            "linear_total": 12582912,
                            "nnz": 558530,
                            "total": 12596224
                        },
                        "20": {
                            "linear_attention_nnz": 115712,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 165888,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 281600,
                            "linear_total": 12582912,
                            "nnz": 287633,
                            "total": 12596224
                        },
                        "21": {
                            "linear_attention_nnz": 151552,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 75776,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 227328,
                            "linear_total": 12582912,
                            "nnz": 233509,
                            "total": 12596224
                        },
                        "22": {
                            "linear_attention_nnz": 46080,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 100352,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 146432,
                            "linear_total": 12582912,
                            "nnz": 152209,
                            "total": 12596224
                        },
                        "23": {
                            "linear_attention_nnz": 24576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 184320,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 208896,
                            "linear_total": 12582912,
                            "nnz": 214394,
                            "total": 12596224
                        },
                        "3": {
                            "linear_attention_nnz": 301056,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 432128,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 733184,
                            "linear_total": 12582912,
                            "nnz": 739891,
                            "total": 12596224
                        },
                        "4": {
                            "linear_attention_nnz": 313344,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 452608,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 765952,
                            "linear_total": 12582912,
                            "nnz": 772669,
                            "total": 12596224
                        },
                        "5": {
                            "linear_attention_nnz": 24576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 614400,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 638976,
                            "linear_total": 12582912,
                            "nnz": 644684,
                            "total": 12596224
                        },
                        "6": {
                            "linear_attention_nnz": 155648,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 598016,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 753664,
                            "linear_total": 12582912,
                            "nnz": 760036,
                            "total": 12596224
                        },
                        "7": {
                            "linear_attention_nnz": 324608,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 466944,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 791552,
                            "linear_total": 12582912,
                            "nnz": 798276,
                            "total": 12596224
                        },
                        "8": {
                            "linear_attention_nnz": 97280,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 673792,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 771072,
                            "linear_total": 12582912,
                            "nnz": 777449,
                            "total": 12596224
                        },
                        "9": {
                            "linear_attention_nnz": 473088,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 692224,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1165312,
                            "linear_total": 12582912,
                            "nnz": 1172306,
                            "total": 12596224
                        }
                    },
                    "linear_nnz": 21481472,
                    "linear_sparsity": 92.88669162326389,
                    "linear_total": 301989888,
                    "nnz": 53428734,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            12,
                            15
                        ],
                        "12": [
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "13": [
                            2,
                            3,
                            4,
                            9,
                            10,
                            11,
                            12,
                            13
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            11,
                            12
                        ],
                        "15": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            8,
                            11,
                            12,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "19": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "21": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "23": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12,
                            13,
                            15
                        ]
                    },
                    "total": 334094338,
                    "total_sparsity": 84.00789001099444
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 10000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test_large/large_regu_60_d0.25",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test_large/large_regu_60_d0.25",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 8,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test_large/large_regu_60_d0.25",
                    "save_steps": 10000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.9851273164745,
            "fill_rate": 0.18968822337962976,
            "speedup": 1.3610235261481995
        },
        {
            "annotate": "7",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-large-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 1024,
                    "initializer_range": 0.02,
                    "intermediate_size": 4096,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 16,
                    "num_hidden_layers": 24,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.120151371807,
                    "f1": 86.51897974152047
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_large/large_regu_60_d0.25_partial/checkpoint-160000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 60
                },
                "speed": {
                    "cuda_eval_elapsed_time": 27.425516578674316,
                    "eval_elapsed_time": 34.279891720972955
                },
                "speedup": 1.4072439764136124,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 336896,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 239616,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 576512,
                            "linear_total": 12582912,
                            "nnz": 583093,
                            "total": 12596224
                        },
                        "1": {
                            "linear_attention_nnz": 104448,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 264192,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 368640,
                            "linear_total": 12582912,
                            "nnz": 374913,
                            "total": 12596224
                        },
                        "10": {
                            "linear_attention_nnz": 692224,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 954368,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1646592,
                            "linear_total": 12582912,
                            "nnz": 1654034,
                            "total": 12596224
                        },
                        "11": {
                            "linear_attention_nnz": 736256,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1071104,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1807360,
                            "linear_total": 12582912,
                            "nnz": 1814891,
                            "total": 12596224
                        },
                        "12": {
                            "linear_attention_nnz": 663552,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1212416,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1875968,
                            "linear_total": 12582912,
                            "nnz": 1883472,
                            "total": 12596224
                        },
                        "13": {
                            "linear_attention_nnz": 1081344,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1165312,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2246656,
                            "linear_total": 12582912,
                            "nnz": 2254745,
                            "total": 12596224
                        },
                        "14": {
                            "linear_attention_nnz": 798720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 978944,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1777664,
                            "linear_total": 12582912,
                            "nnz": 1785502,
                            "total": 12596224
                        },
                        "15": {
                            "linear_attention_nnz": 826368,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 749568,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1575936,
                            "linear_total": 12582912,
                            "nnz": 1583406,
                            "total": 12596224
                        },
                        "16": {
                            "linear_attention_nnz": 796672,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 425984,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1222656,
                            "linear_total": 12582912,
                            "nnz": 1230192,
                            "total": 12596224
                        },
                        "17": {
                            "linear_attention_nnz": 643072,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 374784,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1017856,
                            "linear_total": 12582912,
                            "nnz": 1025399,
                            "total": 12596224
                        },
                        "18": {
                            "linear_attention_nnz": 565248,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 350208,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 915456,
                            "linear_total": 12582912,
                            "nnz": 922667,
                            "total": 12596224
                        },
                        "19": {
                            "linear_attention_nnz": 237568,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 307200,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 544768,
                            "linear_total": 12582912,
                            "nnz": 551094,
                            "total": 12596224
                        },
                        "2": {
                            "linear_attention_nnz": 246784,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 405504,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 652288,
                            "linear_total": 12582912,
                            "nnz": 658918,
                            "total": 12596224
                        },
                        "20": {
                            "linear_attention_nnz": 164864,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 172032,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 336896,
                            "linear_total": 12582912,
                            "nnz": 343124,
                            "total": 12596224
                        },
                        "21": {
                            "linear_attention_nnz": 164864,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 106496,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 271360,
                            "linear_total": 12582912,
                            "nnz": 277460,
                            "total": 12596224
                        },
                        "22": {
                            "linear_attention_nnz": 32768,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 112640,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 145408,
                            "linear_total": 12582912,
                            "nnz": 150967,
                            "total": 12596224
                        },
                        "23": {
                            "linear_attention_nnz": 33792,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 186368,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 220160,
                            "linear_total": 12582912,
                            "nnz": 225755,
                            "total": 12596224
                        },
                        "3": {
                            "linear_attention_nnz": 311296,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 514048,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 825344,
                            "linear_total": 12582912,
                            "nnz": 832123,
                            "total": 12596224
                        },
                        "4": {
                            "linear_attention_nnz": 349184,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 530432,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 879616,
                            "linear_total": 12582912,
                            "nnz": 886371,
                            "total": 12596224
                        },
                        "5": {
                            "linear_attention_nnz": 24576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 628736,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 653312,
                            "linear_total": 12582912,
                            "nnz": 659027,
                            "total": 12596224
                        },
                        "6": {
                            "linear_attention_nnz": 171008,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 659456,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 830464,
                            "linear_total": 12582912,
                            "nnz": 836994,
                            "total": 12596224
                        },
                        "7": {
                            "linear_attention_nnz": 315392,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 495616,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 811008,
                            "linear_total": 12582912,
                            "nnz": 817778,
                            "total": 12596224
                        },
                        "8": {
                            "linear_attention_nnz": 120832,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 712704,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 833536,
                            "linear_total": 12582912,
                            "nnz": 839900,
                            "total": 12596224
                        },
                        "9": {
                            "linear_attention_nnz": 504832,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 776192,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1281024,
                            "linear_total": 12582912,
                            "nnz": 1288059,
                            "total": 12596224
                        }
                    },
                    "linear_nnz": 23316480,
                    "linear_sparsity": 92.279052734375,
                    "linear_total": 301989888,
                    "nnz": 55264846,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            1,
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            12,
                            15
                        ],
                        "12": [
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "13": [
                            2,
                            3,
                            4,
                            6,
                            10,
                            11,
                            12,
                            13
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            11,
                            14
                        ],
                        "15": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            8,
                            11,
                            12,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "19": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "21": [
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "23": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12,
                            13,
                            15
                        ]
                    },
                    "total": 334094338,
                    "total_sparsity": 83.45831110732563
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 10000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test_large/large_regu_60_d0.25",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test_large/large_regu_60_d0.25",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 8,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test_large/large_regu_60_d0.25",
                    "save_steps": 10000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.51897974152047,
            "fill_rate": 0.20589192708333331,
            "speedup": 1.4072439764136124
        },
        {
            "annotate": "7",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-large-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 1024,
                    "initializer_range": 0.02,
                    "intermediate_size": 4096,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 16,
                    "num_hidden_layers": 24,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.72280037842951,
                    "f1": 86.42554404823127
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_large/large_regu_60_d0.25_partial/checkpoint-170000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 60
                },
                "speed": {
                    "cuda_eval_elapsed_time": 26.95893399810791,
                    "eval_elapsed_time": 34.00091099366546
                },
                "speedup": 1.4315993728861762,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 328704,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 233472,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 562176,
                            "linear_total": 12582912,
                            "nnz": 568786,
                            "total": 12596224
                        },
                        "1": {
                            "linear_attention_nnz": 106496,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 256000,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 362496,
                            "linear_total": 12582912,
                            "nnz": 368861,
                            "total": 12596224
                        },
                        "10": {
                            "linear_attention_nnz": 734208,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 937984,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1672192,
                            "linear_total": 12582912,
                            "nnz": 1679626,
                            "total": 12596224
                        },
                        "11": {
                            "linear_attention_nnz": 739328,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1062912,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1802240,
                            "linear_total": 12582912,
                            "nnz": 1809767,
                            "total": 12596224
                        },
                        "12": {
                            "linear_attention_nnz": 655360,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1189888,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1845248,
                            "linear_total": 12582912,
                            "nnz": 1852741,
                            "total": 12596224
                        },
                        "13": {
                            "linear_attention_nnz": 1070080,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1142784,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2212864,
                            "linear_total": 12582912,
                            "nnz": 2220942,
                            "total": 12596224
                        },
                        "14": {
                            "linear_attention_nnz": 719872,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 950272,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1670144,
                            "linear_total": 12582912,
                            "nnz": 1677744,
                            "total": 12596224
                        },
                        "15": {
                            "linear_attention_nnz": 798720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 733184,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1531904,
                            "linear_total": 12582912,
                            "nnz": 1539366,
                            "total": 12596224
                        },
                        "16": {
                            "linear_attention_nnz": 747520,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 419840,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1167360,
                            "linear_total": 12582912,
                            "nnz": 1174829,
                            "total": 12596224
                        },
                        "17": {
                            "linear_attention_nnz": 615424,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 370688,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 986112,
                            "linear_total": 12582912,
                            "nnz": 993589,
                            "total": 12596224
                        },
                        "18": {
                            "linear_attention_nnz": 537600,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 348160,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 885760,
                            "linear_total": 12582912,
                            "nnz": 892842,
                            "total": 12596224
                        },
                        "19": {
                            "linear_attention_nnz": 211968,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 305152,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 517120,
                            "linear_total": 12582912,
                            "nnz": 523477,
                            "total": 12596224
                        },
                        "2": {
                            "linear_attention_nnz": 240640,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 397312,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 637952,
                            "linear_total": 12582912,
                            "nnz": 644578,
                            "total": 12596224
                        },
                        "20": {
                            "linear_attention_nnz": 158720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 169984,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 328704,
                            "linear_total": 12582912,
                            "nnz": 334931,
                            "total": 12596224
                        },
                        "21": {
                            "linear_attention_nnz": 132096,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 104448,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 236544,
                            "linear_total": 12582912,
                            "nnz": 242675,
                            "total": 12596224
                        },
                        "22": {
                            "linear_attention_nnz": 35840,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 110592,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 146432,
                            "linear_total": 12582912,
                            "nnz": 152022,
                            "total": 12596224
                        },
                        "23": {
                            "linear_attention_nnz": 27648,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 178176,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 205824,
                            "linear_total": 12582912,
                            "nnz": 211415,
                            "total": 12596224
                        },
                        "3": {
                            "linear_attention_nnz": 314368,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 499712,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 814080,
                            "linear_total": 12582912,
                            "nnz": 820820,
                            "total": 12596224
                        },
                        "4": {
                            "linear_attention_nnz": 347136,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 507904,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 855040,
                            "linear_total": 12582912,
                            "nnz": 861816,
                            "total": 12596224
                        },
                        "5": {
                            "linear_attention_nnz": 24576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 614400,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 638976,
                            "linear_total": 12582912,
                            "nnz": 644684,
                            "total": 12596224
                        },
                        "6": {
                            "linear_attention_nnz": 161792,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 645120,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 806912,
                            "linear_total": 12582912,
                            "nnz": 813435,
                            "total": 12596224
                        },
                        "7": {
                            "linear_attention_nnz": 325632,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 481280,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 806912,
                            "linear_total": 12582912,
                            "nnz": 813707,
                            "total": 12596224
                        },
                        "8": {
                            "linear_attention_nnz": 101376,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 692224,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 793600,
                            "linear_total": 12582912,
                            "nnz": 799858,
                            "total": 12596224
                        },
                        "9": {
                            "linear_attention_nnz": 475136,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 765952,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1241088,
                            "linear_total": 12582912,
                            "nnz": 1248118,
                            "total": 12596224
                        }
                    },
                    "linear_nnz": 22727680,
                    "linear_sparsity": 92.47402615017361,
                    "linear_total": 301989888,
                    "nnz": 54675591,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            1,
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            12,
                            15
                        ],
                        "12": [
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "13": [
                            2,
                            3,
                            4,
                            6,
                            10,
                            11,
                            12,
                            13
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            10,
                            11,
                            14
                        ],
                        "15": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            8,
                            11,
                            12,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "19": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "21": [
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "23": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12,
                            13,
                            15
                        ]
                    },
                    "total": 334094338,
                    "total_sparsity": 83.63468494338866
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 10000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test_large/large_regu_60_d0.25",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test_large/large_regu_60_d0.25",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 8,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test_large/large_regu_60_d0.25",
                    "save_steps": 10000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.42554404823127,
            "fill_rate": 0.20069263599537024,
            "speedup": 1.4315993728861762
        },
        {
            "annotate": "7",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-large-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 1024,
                    "initializer_range": 0.02,
                    "intermediate_size": 4096,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 16,
                    "num_hidden_layers": 24,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.62819299905392,
                    "f1": 86.34346487920875
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_large/large_regu_60_d0.25_partial/checkpoint-180000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 60
                },
                "speed": {
                    "cuda_eval_elapsed_time": 26.94367268371582,
                    "eval_elapsed_time": 34.14706540107727
                },
                "speedup": 1.4324102529903697,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 332800,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 225280,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 558080,
                            "linear_total": 12582912,
                            "nnz": 564654,
                            "total": 12596224
                        },
                        "1": {
                            "linear_attention_nnz": 95232,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 247808,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 343040,
                            "linear_total": 12582912,
                            "nnz": 349401,
                            "total": 12596224
                        },
                        "10": {
                            "linear_attention_nnz": 688128,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 921600,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1609728,
                            "linear_total": 12582912,
                            "nnz": 1617154,
                            "total": 12596224
                        },
                        "11": {
                            "linear_attention_nnz": 708608,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1050624,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1759232,
                            "linear_total": 12582912,
                            "nnz": 1766753,
                            "total": 12596224
                        },
                        "12": {
                            "linear_attention_nnz": 659456,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1167360,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1826816,
                            "linear_total": 12582912,
                            "nnz": 1834298,
                            "total": 12596224
                        },
                        "13": {
                            "linear_attention_nnz": 984064,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1122304,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2106368,
                            "linear_total": 12582912,
                            "nnz": 2114436,
                            "total": 12596224
                        },
                        "14": {
                            "linear_attention_nnz": 730112,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 927744,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1657856,
                            "linear_total": 12582912,
                            "nnz": 1665413,
                            "total": 12596224
                        },
                        "15": {
                            "linear_attention_nnz": 799744,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 720896,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1520640,
                            "linear_total": 12582912,
                            "nnz": 1528064,
                            "total": 12596224
                        },
                        "16": {
                            "linear_attention_nnz": 738304,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 413696,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1152000,
                            "linear_total": 12582912,
                            "nnz": 1159530,
                            "total": 12596224
                        },
                        "17": {
                            "linear_attention_nnz": 640000,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 370688,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1010688,
                            "linear_total": 12582912,
                            "nnz": 1018229,
                            "total": 12596224
                        },
                        "18": {
                            "linear_attention_nnz": 534528,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 346112,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 880640,
                            "linear_total": 12582912,
                            "nnz": 887785,
                            "total": 12596224
                        },
                        "19": {
                            "linear_attention_nnz": 217088,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 305152,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 522240,
                            "linear_total": 12582912,
                            "nnz": 528501,
                            "total": 12596224
                        },
                        "2": {
                            "linear_attention_nnz": 251904,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 391168,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 643072,
                            "linear_total": 12582912,
                            "nnz": 649695,
                            "total": 12596224
                        },
                        "20": {
                            "linear_attention_nnz": 161792,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 169984,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 331776,
                            "linear_total": 12582912,
                            "nnz": 337939,
                            "total": 12596224
                        },
                        "21": {
                            "linear_attention_nnz": 153600,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 100352,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 253952,
                            "linear_total": 12582912,
                            "nnz": 260049,
                            "total": 12596224
                        },
                        "22": {
                            "linear_attention_nnz": 30720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 108544,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 139264,
                            "linear_total": 12582912,
                            "nnz": 144821,
                            "total": 12596224
                        },
                        "23": {
                            "linear_attention_nnz": 29696,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 172032,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 201728,
                            "linear_total": 12582912,
                            "nnz": 207316,
                            "total": 12596224
                        },
                        "3": {
                            "linear_attention_nnz": 287744,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 493568,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 781312,
                            "linear_total": 12582912,
                            "nnz": 788017,
                            "total": 12596224
                        },
                        "4": {
                            "linear_attention_nnz": 305152,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 491520,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 796672,
                            "linear_total": 12582912,
                            "nnz": 803440,
                            "total": 12596224
                        },
                        "5": {
                            "linear_attention_nnz": 24576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 602112,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 626688,
                            "linear_total": 12582912,
                            "nnz": 632390,
                            "total": 12596224
                        },
                        "6": {
                            "linear_attention_nnz": 163840,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 620544,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 784384,
                            "linear_total": 12582912,
                            "nnz": 790895,
                            "total": 12596224
                        },
                        "7": {
                            "linear_attention_nnz": 301056,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 471040,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 772096,
                            "linear_total": 12582912,
                            "nnz": 778694,
                            "total": 12596224
                        },
                        "8": {
                            "linear_attention_nnz": 112640,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 684032,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 796672,
                            "linear_total": 12582912,
                            "nnz": 803150,
                            "total": 12596224
                        },
                        "9": {
                            "linear_attention_nnz": 484352,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 747520,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1231872,
                            "linear_total": 12582912,
                            "nnz": 1238893,
                            "total": 12596224
                        }
                    },
                    "linear_nnz": 22306816,
                    "linear_sparsity": 92.61338975694444,
                    "linear_total": 301989888,
                    "nnz": 54254479,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            1,
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            12,
                            15
                        ],
                        "12": [
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "13": [
                            2,
                            3,
                            4,
                            6,
                            10,
                            11,
                            12,
                            13
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            10,
                            11,
                            14
                        ],
                        "15": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            8,
                            11,
                            12,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "19": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "21": [
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "23": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12,
                            13,
                            15
                        ]
                    },
                    "total": 334094338,
                    "total_sparsity": 83.76073077898135
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 10000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test_large/large_regu_60_d0.25",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test_large/large_regu_60_d0.25",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 8,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test_large/large_regu_60_d0.25",
                    "save_steps": 10000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.34346487920875,
            "fill_rate": 0.1969762731481482,
            "speedup": 1.4324102529903697
        }
    ],
    "Block/struct method, bs= 64x32, v=0, s=b": [
        {
            "annotate": "22",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.78902554399244,
                    "f1": 86.64151988736798
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test2/hp_od-output__squad_test2_es-steps_nte20_ls250_est5000_rn-output__squad_test2_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_abc32_it0_fw10_r-l1_rfl10_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 64,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 15.454076248168946,
                    "eval_elapsed_time": 22.627552575897425
                },
                "speedup": 2.4973600741709747,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 543488,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1016832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1560320,
                            "linear_total": 7077888,
                            "nnz": 1570304,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 593664,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1222656,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1816320,
                            "linear_total": 7077888,
                            "nnz": 1826304,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 409088,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 178176,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 587264,
                            "linear_total": 7077888,
                            "nnz": 597248,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 250880,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 446976,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 697856,
                            "linear_total": 7077888,
                            "nnz": 707840,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 603904,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1420800,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2024704,
                            "linear_total": 7077888,
                            "nnz": 2034688,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 870656,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1459200,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2329856,
                            "linear_total": 7077888,
                            "nnz": 2339840,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 887552,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1445376,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2332928,
                            "linear_total": 7077888,
                            "nnz": 2342912,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 720640,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1370112,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2090752,
                            "linear_total": 7077888,
                            "nnz": 2100736,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 806400,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1081344,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1887744,
                            "linear_total": 7077888,
                            "nnz": 1897728,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 926464,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 815616,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1742080,
                            "linear_total": 7077888,
                            "nnz": 1752064,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 455936,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 488448,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 944384,
                            "linear_total": 7077888,
                            "nnz": 954368,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 505600,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 199680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 705280,
                            "linear_total": 7077888,
                            "nnz": 715264,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 18719488,
                    "linear_sparsity": 77.96012972608024,
                    "linear_total": 84934656,
                    "nnz": 42678018,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            3,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 60.80744850279245
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test2",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "output_dir": "output/squad_test2",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test2",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 78.94985808893094,
                    "f1": 86.83052028636654
                }
            },
            "f1": 86.64151988736798,
            "fill_rate": 0.2203987027391976,
            "speedup": 2.4973600741709747
        },
        {
            "annotate": "22",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.93093661305582,
                    "f1": 86.85787750084084
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test2/hp_od-output__squad_test2_es-steps_nte20_ls250_est5000_rn-output__squad_test2_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_abc32_it0_fw10_r-l1_rfl10_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-85000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 64,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 15.710145233154297,
                    "eval_elapsed_time": 22.9045692961663
                },
                "speedup": 2.456654119525417,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 579328,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1081344,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1660672,
                            "linear_total": 7077888,
                            "nnz": 1670656,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 632576,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1267200,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1899776,
                            "linear_total": 7077888,
                            "nnz": 1909760,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 448256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 182784,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 631040,
                            "linear_total": 7077888,
                            "nnz": 641024,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 288256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 462336,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 750592,
                            "linear_total": 7077888,
                            "nnz": 760576,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 584192,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1446912,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2031104,
                            "linear_total": 7077888,
                            "nnz": 2041088,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1049600,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1494528,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2544128,
                            "linear_total": 7077888,
                            "nnz": 2554112,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 916736,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1479168,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2395904,
                            "linear_total": 7077888,
                            "nnz": 2405888,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 790272,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1394688,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2184960,
                            "linear_total": 7077888,
                            "nnz": 2194944,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 798720,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1113600,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1912320,
                            "linear_total": 7077888,
                            "nnz": 1922304,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 969216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 837120,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1806336,
                            "linear_total": 7077888,
                            "nnz": 1816320,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 471808,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 497664,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 969472,
                            "linear_total": 7077888,
                            "nnz": 979456,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 505344,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 211968,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 717312,
                            "linear_total": 7077888,
                            "nnz": 727296,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 19503616,
                    "linear_sparsity": 77.03691647376543,
                    "linear_total": 84934656,
                    "nnz": 43462146,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 60.08735936884057
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test2",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "output_dir": "output/squad_test2",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test2",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 79.15799432355723,
                    "f1": 87.0225802715423
                }
            },
            "f1": 86.85787750084084,
            "fill_rate": 0.22963083526234573,
            "speedup": 2.456654119525417
        }
    ],
    "Block/struct method, final fine tuned, s=b": [
        {
            "annotate": "27",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.69063386944181,
                    "f1": 88.06386432532665
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_aws_nn-pruning-v10-a32-l10-dl0-5--2021-01-21--00-54-13/checkpoint-22132",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v10-a32-l10-dl0-5--2021-01-21--00-54-13/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 17.390718185424806,
                    "eval_elapsed_time": 24.534384376835078
                },
                "speedup": 2.2192523962418718,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 634368,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1420800,
                            "linear_total": 7077888,
                            "nnz": 1426589,
                            "total": 7086336
                        },
                        "1": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 916992,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1703424,
                            "linear_total": 7077888,
                            "nnz": 1709397,
                            "total": 7086336
                        },
                        "10": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 112128,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1291776,
                            "linear_total": 7077888,
                            "nnz": 1297609,
                            "total": 7086720
                        },
                        "11": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 313344,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 903168,
                            "linear_total": 7077888,
                            "nnz": 908556,
                            "total": 7086144
                        },
                        "2": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1016832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2786304,
                            "linear_total": 7077888,
                            "nnz": 2793302,
                            "total": 7087296
                        },
                        "3": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1076736,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2649600,
                            "linear_total": 7077888,
                            "nnz": 2656445,
                            "total": 7087104
                        },
                        "4": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1158144,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3124224,
                            "linear_total": 7077888,
                            "nnz": 3131506,
                            "total": 7087488
                        },
                        "5": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1073664,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2449920,
                            "linear_total": 7077888,
                            "nnz": 2456571,
                            "total": 7086912
                        },
                        "6": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 815616,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2388480,
                            "linear_total": 7077888,
                            "nnz": 2395155,
                            "total": 7087104
                        },
                        "7": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 629760,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2006016,
                            "linear_total": 7077888,
                            "nnz": 2012378,
                            "total": 7086912
                        },
                        "8": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 337920,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1910784,
                            "linear_total": 7077888,
                            "nnz": 1917148,
                            "total": 7087104
                        },
                        "9": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 139776,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1122816,
                            "linear_total": 7077888,
                            "nnz": 1128475,
                            "total": 7086528
                        }
                    },
                    "linear_nnz": 23757312,
                    "linear_sparsity": 72.0287181712963,
                    "linear_total": 84934656,
                    "nnz": 47671853,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108880706,
                    "total_sparsity": 56.216436546618276
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_aws_nn-pruning-v10-a32-l10-dl0-5--2021-01-21--00-54-13",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 4,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_aws_nn-pruning-v10-a32-l10-dl0-5--2021-01-21--00-54-13",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_aws_nn-pruning-v10-a32-l10-dl0-5--2021-01-21--00-54-13",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.06386432532665,
            "fill_rate": 0.279712818287037,
            "speedup": 2.2192523962418718
        },
        {
            "annotate": "25",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.23651844843897,
                    "f1": 87.68464122182475
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_aws_nn-pruning-v10-a32-l10-dl1--2021-01-21--00-53-40/checkpoint-22132",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v10-a32-l10-dl1--2021-01-21--00-53-40/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 17.154361824035647,
                    "eval_elapsed_time": 24.304617804009467
                },
                "speedup": 2.249829716853412,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 297984,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1477632,
                            "linear_total": 7077888,
                            "nnz": 1483586,
                            "total": 7086720
                        },
                        "1": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 483840,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1466880,
                            "linear_total": 7077888,
                            "nnz": 1472763,
                            "total": 7086528
                        },
                        "10": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 73728,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1253376,
                            "linear_total": 7077888,
                            "nnz": 1259184,
                            "total": 7086720
                        },
                        "11": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 159744,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 749568,
                            "linear_total": 7077888,
                            "nnz": 754856,
                            "total": 7086144
                        },
                        "2": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 619008,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2388480,
                            "linear_total": 7077888,
                            "nnz": 2395219,
                            "total": 7087296
                        },
                        "3": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 657408,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2230272,
                            "linear_total": 7077888,
                            "nnz": 2236844,
                            "total": 7087104
                        },
                        "4": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 705024,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2671104,
                            "linear_total": 7077888,
                            "nnz": 2678091,
                            "total": 7087488
                        },
                        "5": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 668160,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2241024,
                            "linear_total": 7077888,
                            "nnz": 2247603,
                            "total": 7087104
                        },
                        "6": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 516096,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2088960,
                            "linear_total": 7077888,
                            "nnz": 2095440,
                            "total": 7087104
                        },
                        "7": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 384000,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1760256,
                            "linear_total": 7077888,
                            "nnz": 1766458,
                            "total": 7086912
                        },
                        "8": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 204288,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1973760,
                            "linear_total": 7077888,
                            "nnz": 1980229,
                            "total": 7087296
                        },
                        "9": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 92160,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1271808,
                            "linear_total": 7077888,
                            "nnz": 1277628,
                            "total": 7086720
                        }
                    },
                    "linear_nnz": 21573120,
                    "linear_sparsity": 74.60033275462963,
                    "linear_total": 84934656,
                    "nnz": 45486623,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108881858,
                    "total_sparsity": 58.223873255359024
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_aws_nn-pruning-v10-a32-l10-dl1--2021-01-21--00-53-40",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 4,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_aws_nn-pruning-v10-a32-l10-dl1--2021-01-21--00-53-40",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_aws_nn-pruning-v10-a32-l10-dl1--2021-01-21--00-53-40",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.68464122182475,
            "fill_rate": 0.2539966724537037,
            "speedup": 2.249829716853412
        },
        {
            "annotate": "21",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.4228949858089,
                    "f1": 87.22907143184382
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_aws_nn-pruning-v10-a32-l20-dl0-5--2021-01-21--00-55-44/checkpoint-22132",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v10-a32-l20-dl0-5--2021-01-21--00-55-44/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 14.848762104034424,
                    "eval_elapsed_time": 22.048566517885774
                },
                "speedup": 2.5991656903766382,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 411648,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1198080,
                            "linear_total": 7077888,
                            "nnz": 1203724,
                            "total": 7086336
                        },
                        "1": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 592896,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1379328,
                            "linear_total": 7077888,
                            "nnz": 1385090,
                            "total": 7086336
                        },
                        "10": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 87552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1070592,
                            "linear_total": 7077888,
                            "nnz": 1076217,
                            "total": 7086528
                        },
                        "11": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 199680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 789504,
                            "linear_total": 7077888,
                            "nnz": 794818,
                            "total": 7086144
                        },
                        "2": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 698880,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1878528,
                            "linear_total": 7077888,
                            "nnz": 1884743,
                            "total": 7086720
                        },
                        "3": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 714240,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2090496,
                            "linear_total": 7077888,
                            "nnz": 2096913,
                            "total": 7086912
                        },
                        "4": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 834048,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2210304,
                            "linear_total": 7077888,
                            "nnz": 2216799,
                            "total": 7086912
                        },
                        "5": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 743424,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1726464,
                            "linear_total": 7077888,
                            "nnz": 1732516,
                            "total": 7086528
                        },
                        "6": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 568320,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1747968,
                            "linear_total": 7077888,
                            "nnz": 1754098,
                            "total": 7086720
                        },
                        "7": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 450048,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1826304,
                            "linear_total": 7077888,
                            "nnz": 1832549,
                            "total": 7086912
                        },
                        "8": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 264192,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1443840,
                            "linear_total": 7077888,
                            "nnz": 1449772,
                            "total": 7086720
                        },
                        "9": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 101376,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1084416,
                            "linear_total": 7077888,
                            "nnz": 1090050,
                            "total": 7086528
                        }
                    },
                    "linear_nnz": 18445824,
                    "linear_sparsity": 78.28233506944444,
                    "linear_total": 84934656,
                    "nnz": 42356011,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108878018,
                    "total_sparsity": 61.097738755677945
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_aws_nn-pruning-v10-a32-l20-dl0-5--2021-01-21--00-55-44",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 4,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_aws_nn-pruning-v10-a32-l20-dl0-5--2021-01-21--00-55-44",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_aws_nn-pruning-v10-a32-l20-dl0-5--2021-01-21--00-55-44",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.22907143184382,
            "fill_rate": 0.21717664930555558,
            "speedup": 2.5991656903766382
        },
        {
            "annotate": "19",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2,
                            4,
                            6,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.82686849574267,
                    "f1": 86.75497848244157
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_aws_nn-pruning-v10-a32-l20-dl1--2021-01-21--00-55-15/checkpoint-22132",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v10-a32-l20-dl1--2021-01-21--00-55-15/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 14.354346725463868,
                    "eval_elapsed_time": 21.489493974950165
                },
                "speedup": 2.68869031405704,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 181248,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 967680,
                            "linear_total": 7077888,
                            "nnz": 973174,
                            "total": 7086336
                        },
                        "1": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 299520,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1085952,
                            "linear_total": 7077888,
                            "nnz": 1091523,
                            "total": 7086336
                        },
                        "10": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 58368,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1041408,
                            "linear_total": 7077888,
                            "nnz": 1047014,
                            "total": 7086528
                        },
                        "11": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 96768,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 686592,
                            "linear_total": 7077888,
                            "nnz": 691839,
                            "total": 7086144
                        },
                        "2": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 407040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1586688,
                            "linear_total": 7077888,
                            "nnz": 1592713,
                            "total": 7086720
                        },
                        "3": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 440832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2013696,
                            "linear_total": 7077888,
                            "nnz": 2020127,
                            "total": 7087104
                        },
                        "4": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 496128,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1872384,
                            "linear_total": 7077888,
                            "nnz": 1878659,
                            "total": 7086912
                        },
                        "5": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 433152,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1416192,
                            "linear_total": 7077888,
                            "nnz": 1422042,
                            "total": 7086528
                        },
                        "6": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 337920,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1517568,
                            "linear_total": 7077888,
                            "nnz": 1523548,
                            "total": 7086720
                        },
                        "7": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 268800,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1645056,
                            "linear_total": 7077888,
                            "nnz": 1651183,
                            "total": 7086912
                        },
                        "8": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 158208,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1534464,
                            "linear_total": 7077888,
                            "nnz": 1540519,
                            "total": 7086912
                        },
                        "9": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 73728,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1056768,
                            "linear_total": 7077888,
                            "nnz": 1062384,
                            "total": 7086528
                        }
                    },
                    "linear_nnz": 16424448,
                    "linear_sparsity": 80.66225405092592,
                    "linear_total": 84934656,
                    "nnz": 40333447,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2,
                            4,
                            6,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108878402,
                    "total_sparsity": 62.95551159907728
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_aws_nn-pruning-v10-a32-l20-dl1--2021-01-21--00-55-15",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 4,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_aws_nn-pruning-v10-a32-l20-dl1--2021-01-21--00-55-15",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_aws_nn-pruning-v10-a32-l20-dl1--2021-01-21--00-55-15",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.75497848244157,
            "fill_rate": 0.1933774594907408,
            "speedup": 2.68869031405704
        },
        {
            "annotate": "36",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            5,
                            6,
                            7
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            6,
                            7,
                            8,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            2
                        ],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            2,
                            3,
                            7
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 81.69347209082308,
                    "f1": 88.72194531479171
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_aws_nn-pruning-v10-a32-l5-dl0-5--2021-01-21--00-52-45/checkpoint-22132",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v10-a32-l5-dl0-5--2021-01-21--00-52-45/checkpoint-95000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 20.951393741607667,
                    "eval_elapsed_time": 28.213609586004168
                },
                "speedup": 1.8420919143305463,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1125888,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2502144,
                            "linear_total": 7077888,
                            "nnz": 2508829,
                            "total": 7086912
                        },
                        "1": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1285632,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2268672,
                            "linear_total": 7077888,
                            "nnz": 2275077,
                            "total": 7086528
                        },
                        "10": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 168960,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1545216,
                            "linear_total": 7077888,
                            "nnz": 1551278,
                            "total": 7086912
                        },
                        "11": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 485376,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1468416,
                            "linear_total": 7077888,
                            "nnz": 1474300,
                            "total": 7086528
                        },
                        "2": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1523712,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3293184,
                            "linear_total": 7077888,
                            "nnz": 3300512,
                            "total": 7087296
                        },
                        "3": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1555968,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3325440,
                            "linear_total": 7077888,
                            "nnz": 3332789,
                            "total": 7087296
                        },
                        "4": {
                            "linear_attention_nnz": 2162688,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1617408,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3780096,
                            "linear_total": 7077888,
                            "nnz": 3787869,
                            "total": 7087680
                        },
                        "5": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1514496,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3480576,
                            "linear_total": 7077888,
                            "nnz": 3488090,
                            "total": 7087488
                        },
                        "6": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1135104,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2904576,
                            "linear_total": 7077888,
                            "nnz": 2911651,
                            "total": 7087296
                        },
                        "7": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 847872,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2420736,
                            "linear_total": 7077888,
                            "nnz": 2427432,
                            "total": 7087104
                        },
                        "8": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 474624,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2440704,
                            "linear_total": 7077888,
                            "nnz": 2447541,
                            "total": 7087488
                        },
                        "9": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 208896,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1388544,
                            "linear_total": 7077888,
                            "nnz": 1394440,
                            "total": 7086720
                        }
                    },
                    "linear_nnz": 30818304,
                    "linear_sparsity": 63.71527777777778,
                    "linear_total": 84934656,
                    "nnz": 54738530,
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            5,
                            6,
                            7
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            6,
                            7,
                            8,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            2
                        ],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            2,
                            3,
                            7
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108883970,
                    "total_sparsity": 49.72765045212808
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_aws_nn-pruning-v10-a32-l5-dl0-5--2021-01-21--00-52-45",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 4,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_aws_nn-pruning-v10-a32-l5-dl0-5--2021-01-21--00-52-45",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_aws_nn-pruning-v10-a32-l5-dl0-5--2021-01-21--00-52-45",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.72194531479171,
            "fill_rate": 0.3628472222222222,
            "speedup": 1.8420919143305463
        },
        {
            "annotate": "31",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            7
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            6,
                            7,
                            8,
                            11
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            2
                        ],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            2,
                            3,
                            7
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.86092715231788,
                    "f1": 88.26868699204444
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_aws_nn-pruning-v10-a32-l5-dl1--2021-01-21--00-52-16/checkpoint-20000",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v10-a32-l5-dl1--2021-01-21--00-52-16/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 19.458871116638186,
                    "eval_elapsed_time": 26.62503844080493
                },
                "speedup": 1.98338294004996,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 526848,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1903104,
                            "linear_total": 7077888,
                            "nnz": 1909399,
                            "total": 7086912
                        },
                        "1": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 752640,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1735680,
                            "linear_total": 7077888,
                            "nnz": 1741738,
                            "total": 7086528
                        },
                        "10": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 98304,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1671168,
                            "linear_total": 7077888,
                            "nnz": 1677376,
                            "total": 7087104
                        },
                        "11": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 262656,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1245696,
                            "linear_total": 7077888,
                            "nnz": 1251435,
                            "total": 7086528
                        },
                        "2": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 873984,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2840064,
                            "linear_total": 7077888,
                            "nnz": 2847161,
                            "total": 7087488
                        },
                        "3": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 952320,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2721792,
                            "linear_total": 7077888,
                            "nnz": 2728748,
                            "total": 7087296
                        },
                        "4": {
                            "linear_attention_nnz": 2162688,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1046016,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3208704,
                            "linear_total": 7077888,
                            "nnz": 3216105,
                            "total": 7087680
                        },
                        "5": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 986112,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2952192,
                            "linear_total": 7077888,
                            "nnz": 2959362,
                            "total": 7087488
                        },
                        "6": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 740352,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2509824,
                            "linear_total": 7077888,
                            "nnz": 2516642,
                            "total": 7087296
                        },
                        "7": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 559104,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2131968,
                            "linear_total": 7077888,
                            "nnz": 2138476,
                            "total": 7087104
                        },
                        "8": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 293376,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2259456,
                            "linear_total": 7077888,
                            "nnz": 2266175,
                            "total": 7087488
                        },
                        "9": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 113664,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1293312,
                            "linear_total": 7077888,
                            "nnz": 1299146,
                            "total": 7086720
                        }
                    },
                    "linear_nnz": 26472960,
                    "linear_sparsity": 68.83138020833333,
                    "linear_total": 84934656,
                    "nnz": 50390485,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            7
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            6,
                            7,
                            8,
                            11
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            2
                        ],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            2,
                            3,
                            7
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108884354,
                    "total_sparsity": 53.72109660493554
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_aws_nn-pruning-v10-a32-l5-dl1--2021-01-21--00-52-16",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 4,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_aws_nn-pruning-v10-a32-l5-dl1--2021-01-21--00-52-16",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_aws_nn-pruning-v10-a32-l5-dl1--2021-01-21--00-52-16",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.26868699204444,
            "fill_rate": 0.31168619791666674,
            "speedup": 1.98338294004996
        },
        {
            "annotate": "31",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            7
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            6,
                            7,
                            8,
                            11
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            2
                        ],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            2,
                            3,
                            7
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.87038789025544,
                    "f1": 88.24613086360249
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_aws_nn-pruning-v10-a32-l5-dl1--2021-01-21--00-52-16/checkpoint-22132",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v10-a32-l5-dl1--2021-01-21--00-52-16/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 19.453059474945068,
                    "eval_elapsed_time": 26.577815205790102
                },
                "speedup": 1.9839754797994356,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 526848,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1903104,
                            "linear_total": 7077888,
                            "nnz": 1909399,
                            "total": 7086912
                        },
                        "1": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 752640,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1735680,
                            "linear_total": 7077888,
                            "nnz": 1741738,
                            "total": 7086528
                        },
                        "10": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 98304,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1671168,
                            "linear_total": 7077888,
                            "nnz": 1677376,
                            "total": 7087104
                        },
                        "11": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 262656,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1245696,
                            "linear_total": 7077888,
                            "nnz": 1251435,
                            "total": 7086528
                        },
                        "2": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 873984,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2840064,
                            "linear_total": 7077888,
                            "nnz": 2847161,
                            "total": 7087488
                        },
                        "3": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 952320,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2721792,
                            "linear_total": 7077888,
                            "nnz": 2728748,
                            "total": 7087296
                        },
                        "4": {
                            "linear_attention_nnz": 2162688,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1046016,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3208704,
                            "linear_total": 7077888,
                            "nnz": 3216105,
                            "total": 7087680
                        },
                        "5": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 986112,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2952192,
                            "linear_total": 7077888,
                            "nnz": 2959362,
                            "total": 7087488
                        },
                        "6": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 740352,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2509824,
                            "linear_total": 7077888,
                            "nnz": 2516642,
                            "total": 7087296
                        },
                        "7": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 559104,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2131968,
                            "linear_total": 7077888,
                            "nnz": 2138476,
                            "total": 7087104
                        },
                        "8": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 293376,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2259456,
                            "linear_total": 7077888,
                            "nnz": 2266175,
                            "total": 7087488
                        },
                        "9": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 113664,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1293312,
                            "linear_total": 7077888,
                            "nnz": 1299146,
                            "total": 7086720
                        }
                    },
                    "linear_nnz": 26472960,
                    "linear_sparsity": 68.83138020833333,
                    "linear_total": 84934656,
                    "nnz": 50390485,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            7
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            6,
                            7,
                            8,
                            11
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            2
                        ],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            2,
                            3,
                            7
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108884354,
                    "total_sparsity": 53.72109660493554
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_aws_nn-pruning-v10-a32-l5-dl1--2021-01-21--00-52-16",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 4,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_aws_nn-pruning-v10-a32-l5-dl1--2021-01-21--00-52-16",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_aws_nn-pruning-v10-a32-l5-dl1--2021-01-21--00-52-16",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.24613086360249,
            "fill_rate": 0.31168619791666674,
            "speedup": 1.9839754797994356
        },
        {
            "annotate": "33",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            11,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.80416272469253,
                    "f1": 88.20260662536118
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl10_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-15000",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl10_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-90000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 18.439563426971436,
                    "eval_elapsed_time": 25.7331585730426
                },
                "speedup": 2.0930209740713988,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1339392,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2125824,
                            "linear_total": 7077888,
                            "nnz": 2132072,
                            "total": 7086336
                        },
                        "1": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1571328,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2357760,
                            "linear_total": 7077888,
                            "nnz": 2364159,
                            "total": 7086336
                        },
                        "10": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 187392,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1367040,
                            "linear_total": 7077888,
                            "nnz": 1372922,
                            "total": 7086720
                        },
                        "11": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 574464,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1164288,
                            "linear_total": 7077888,
                            "nnz": 1169846,
                            "total": 7086144
                        },
                        "2": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1744896,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3317760,
                            "linear_total": 7077888,
                            "nnz": 3325040,
                            "total": 7087104
                        },
                        "3": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1761792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3334656,
                            "linear_total": 7077888,
                            "nnz": 3341947,
                            "total": 7087104
                        },
                        "4": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1726464,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3495936,
                            "linear_total": 7077888,
                            "nnz": 3503396,
                            "total": 7087296
                        },
                        "5": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1629696,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2809344,
                            "linear_total": 7077888,
                            "nnz": 2816165,
                            "total": 7086720
                        },
                        "6": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1270272,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2646528,
                            "linear_total": 7077888,
                            "nnz": 2653307,
                            "total": 7086912
                        },
                        "7": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 987648,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2363904,
                            "linear_total": 7077888,
                            "nnz": 2370499,
                            "total": 7086912
                        },
                        "8": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 546816,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2119680,
                            "linear_total": 7077888,
                            "nnz": 2126180,
                            "total": 7087104
                        },
                        "9": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 248832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1428480,
                            "linear_total": 7077888,
                            "nnz": 1434402,
                            "total": 7086720
                        }
                    },
                    "linear_nnz": 28531200,
                    "linear_sparsity": 66.40805844907408,
                    "linear_total": 84934656,
                    "nnz": 52448657,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            11,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108880130,
                    "total_sparsity": 51.82899120344548
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl10_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 4,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl10_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl10_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.20260662536118,
            "fill_rate": 0.3359194155092592,
            "speedup": 2.0930209740713988
        },
        {
            "annotate": "33",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            11,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.6717123935667,
                    "f1": 88.128983727943
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl10_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-20000",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl10_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-90000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 18.875869693756105,
                    "eval_elapsed_time": 26.023085076361895
                },
                "speedup": 2.044641843344449,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1339392,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2125824,
                            "linear_total": 7077888,
                            "nnz": 2132072,
                            "total": 7086336
                        },
                        "1": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1571328,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2357760,
                            "linear_total": 7077888,
                            "nnz": 2364159,
                            "total": 7086336
                        },
                        "10": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 187392,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1367040,
                            "linear_total": 7077888,
                            "nnz": 1372922,
                            "total": 7086720
                        },
                        "11": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 574464,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1164288,
                            "linear_total": 7077888,
                            "nnz": 1169846,
                            "total": 7086144
                        },
                        "2": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1744896,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3317760,
                            "linear_total": 7077888,
                            "nnz": 3325040,
                            "total": 7087104
                        },
                        "3": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1761792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3334656,
                            "linear_total": 7077888,
                            "nnz": 3341947,
                            "total": 7087104
                        },
                        "4": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1726464,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3495936,
                            "linear_total": 7077888,
                            "nnz": 3503396,
                            "total": 7087296
                        },
                        "5": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1629696,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2809344,
                            "linear_total": 7077888,
                            "nnz": 2816165,
                            "total": 7086720
                        },
                        "6": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1270272,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2646528,
                            "linear_total": 7077888,
                            "nnz": 2653307,
                            "total": 7086912
                        },
                        "7": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 987648,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2363904,
                            "linear_total": 7077888,
                            "nnz": 2370499,
                            "total": 7086912
                        },
                        "8": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 546816,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2119680,
                            "linear_total": 7077888,
                            "nnz": 2126180,
                            "total": 7087104
                        },
                        "9": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 248832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1428480,
                            "linear_total": 7077888,
                            "nnz": 1434402,
                            "total": 7086720
                        }
                    },
                    "linear_nnz": 28531200,
                    "linear_sparsity": 66.40805844907408,
                    "linear_total": 84934656,
                    "nnz": 52448657,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            11,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108880130,
                    "total_sparsity": 51.82899120344548
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl10_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 4,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl10_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl10_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.128983727943,
            "fill_rate": 0.3359194155092592,
            "speedup": 2.044641843344449
        },
        {
            "annotate": "33",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            11,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.68117313150425,
                    "f1": 88.11014400914335
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl10_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-22132",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl10_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-90000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 18.42703369522095,
                    "eval_elapsed_time": 25.61402732366696
                },
                "speedup": 2.094444154371984,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1339392,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2125824,
                            "linear_total": 7077888,
                            "nnz": 2132072,
                            "total": 7086336
                        },
                        "1": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1571328,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2357760,
                            "linear_total": 7077888,
                            "nnz": 2364159,
                            "total": 7086336
                        },
                        "10": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 187392,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1367040,
                            "linear_total": 7077888,
                            "nnz": 1372922,
                            "total": 7086720
                        },
                        "11": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 574464,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1164288,
                            "linear_total": 7077888,
                            "nnz": 1169846,
                            "total": 7086144
                        },
                        "2": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1744896,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3317760,
                            "linear_total": 7077888,
                            "nnz": 3325040,
                            "total": 7087104
                        },
                        "3": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1761792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3334656,
                            "linear_total": 7077888,
                            "nnz": 3341947,
                            "total": 7087104
                        },
                        "4": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1726464,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3495936,
                            "linear_total": 7077888,
                            "nnz": 3503396,
                            "total": 7087296
                        },
                        "5": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1629696,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2809344,
                            "linear_total": 7077888,
                            "nnz": 2816165,
                            "total": 7086720
                        },
                        "6": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1270272,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2646528,
                            "linear_total": 7077888,
                            "nnz": 2653307,
                            "total": 7086912
                        },
                        "7": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 987648,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2363904,
                            "linear_total": 7077888,
                            "nnz": 2370499,
                            "total": 7086912
                        },
                        "8": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 546816,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2119680,
                            "linear_total": 7077888,
                            "nnz": 2126180,
                            "total": 7087104
                        },
                        "9": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 248832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1428480,
                            "linear_total": 7077888,
                            "nnz": 1434402,
                            "total": 7086720
                        }
                    },
                    "linear_nnz": 28531200,
                    "linear_sparsity": 66.40805844907408,
                    "linear_total": 84934656,
                    "nnz": 52448657,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            11,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108880130,
                    "total_sparsity": 51.82899120344548
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl10_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 4,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl10_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl10_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.11014400914335,
            "fill_rate": 0.3359194155092592,
            "speedup": 2.094444154371984
        },
        {
            "annotate": "26",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.01892147587512,
                    "f1": 87.70568682399205
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl20_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-20000",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl20_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-90000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 15.845825397491456,
                    "eval_elapsed_time": 23.001069764140993
                },
                "speedup": 2.4356189745395627,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 847872,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1634304,
                            "linear_total": 7077888,
                            "nnz": 1640232,
                            "total": 7086336
                        },
                        "1": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1101312,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1887744,
                            "linear_total": 7077888,
                            "nnz": 1893837,
                            "total": 7086336
                        },
                        "10": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 147456,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1130496,
                            "linear_total": 7077888,
                            "nnz": 1136160,
                            "total": 7086528
                        },
                        "11": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 365568,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 955392,
                            "linear_total": 7077888,
                            "nnz": 960814,
                            "total": 7086144
                        },
                        "2": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1221120,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2400768,
                            "linear_total": 7077888,
                            "nnz": 2407323,
                            "total": 7086720
                        },
                        "3": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1211904,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2588160,
                            "linear_total": 7077888,
                            "nnz": 2594901,
                            "total": 7086912
                        },
                        "4": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1279488,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2655744,
                            "linear_total": 7077888,
                            "nnz": 2662529,
                            "total": 7086912
                        },
                        "5": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1216512,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2199552,
                            "linear_total": 7077888,
                            "nnz": 2205912,
                            "total": 7086528
                        },
                        "6": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 952320,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2131968,
                            "linear_total": 7077888,
                            "nnz": 2138348,
                            "total": 7086720
                        },
                        "7": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 715776,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2092032,
                            "linear_total": 7077888,
                            "nnz": 2098450,
                            "total": 7086912
                        },
                        "8": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 434688,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1417728,
                            "linear_total": 7077888,
                            "nnz": 1423579,
                            "total": 7086528
                        },
                        "9": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 172032,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1155072,
                            "linear_total": 7077888,
                            "nnz": 1160752,
                            "total": 7086528
                        }
                    },
                    "linear_nnz": 22248960,
                    "linear_sparsity": 73.80461516203704,
                    "linear_total": 84934656,
                    "nnz": 46161559,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108877826,
                    "total_sparsity": 57.602424023418685
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl20_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 4,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl20_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl20_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.70568682399205,
            "fill_rate": 0.26195384837962965,
            "speedup": 2.4356189745395627
        },
        {
            "annotate": "26",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.02838221381268,
                    "f1": 87.70940223967354
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl20_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-22132",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl20_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-90000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 15.838374267578125,
                    "eval_elapsed_time": 22.999519595876336
                },
                "speedup": 2.436764806371294,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 847872,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1634304,
                            "linear_total": 7077888,
                            "nnz": 1640232,
                            "total": 7086336
                        },
                        "1": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1101312,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1887744,
                            "linear_total": 7077888,
                            "nnz": 1893837,
                            "total": 7086336
                        },
                        "10": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 147456,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1130496,
                            "linear_total": 7077888,
                            "nnz": 1136160,
                            "total": 7086528
                        },
                        "11": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 365568,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 955392,
                            "linear_total": 7077888,
                            "nnz": 960814,
                            "total": 7086144
                        },
                        "2": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1221120,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2400768,
                            "linear_total": 7077888,
                            "nnz": 2407323,
                            "total": 7086720
                        },
                        "3": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1211904,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2588160,
                            "linear_total": 7077888,
                            "nnz": 2594901,
                            "total": 7086912
                        },
                        "4": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1279488,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2655744,
                            "linear_total": 7077888,
                            "nnz": 2662529,
                            "total": 7086912
                        },
                        "5": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1216512,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2199552,
                            "linear_total": 7077888,
                            "nnz": 2205912,
                            "total": 7086528
                        },
                        "6": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 952320,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2131968,
                            "linear_total": 7077888,
                            "nnz": 2138348,
                            "total": 7086720
                        },
                        "7": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 715776,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2092032,
                            "linear_total": 7077888,
                            "nnz": 2098450,
                            "total": 7086912
                        },
                        "8": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 434688,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1417728,
                            "linear_total": 7077888,
                            "nnz": 1423579,
                            "total": 7086528
                        },
                        "9": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 172032,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1155072,
                            "linear_total": 7077888,
                            "nnz": 1160752,
                            "total": 7086528
                        }
                    },
                    "linear_nnz": 22248960,
                    "linear_sparsity": 73.80461516203704,
                    "linear_total": 84934656,
                    "nnz": 46161559,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108877826,
                    "total_sparsity": 57.602424023418685
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl20_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 4,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl20_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl20_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.70940223967354,
            "fill_rate": 0.26195384837962965,
            "speedup": 2.436764806371294
        },
        {
            "annotate": "20",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "3": [
                            2,
                            3,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            8,
                            9,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            6,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.63765373699148,
                    "f1": 86.69392512957342
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl40_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-22132",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl40_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 13.783753513336181,
                    "eval_elapsed_time": 20.85535095212981
                },
                "speedup": 2.799991523936488,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 482304,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1268736,
                            "linear_total": 7077888,
                            "nnz": 1274426,
                            "total": 7086336
                        },
                        "1": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 706560,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1296384,
                            "linear_total": 7077888,
                            "nnz": 1302028,
                            "total": 7086144
                        },
                        "10": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 121344,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1104384,
                            "linear_total": 7077888,
                            "nnz": 1110031,
                            "total": 7086528
                        },
                        "11": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 215040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 804864,
                            "linear_total": 7077888,
                            "nnz": 810188,
                            "total": 7086144
                        },
                        "2": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 850944,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1440768,
                            "linear_total": 7077888,
                            "nnz": 1446506,
                            "total": 7086144
                        },
                        "3": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 826368,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2006016,
                            "linear_total": 7077888,
                            "nnz": 2012314,
                            "total": 7086720
                        },
                        "4": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 923136,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1709568,
                            "linear_total": 7077888,
                            "nnz": 1715545,
                            "total": 7086336
                        },
                        "5": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 880128,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1863168,
                            "linear_total": 7077888,
                            "nnz": 1869309,
                            "total": 7086528
                        },
                        "6": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 645120,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1628160,
                            "linear_total": 7077888,
                            "nnz": 1634148,
                            "total": 7086528
                        },
                        "7": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 525312,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1901568,
                            "linear_total": 7077888,
                            "nnz": 1907862,
                            "total": 7086912
                        },
                        "8": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 333312,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 923136,
                            "linear_total": 7077888,
                            "nnz": 928537,
                            "total": 7086144
                        },
                        "9": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 113664,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1096704,
                            "linear_total": 7077888,
                            "nnz": 1102346,
                            "total": 7086528
                        }
                    },
                    "linear_nnz": 17043456,
                    "linear_sparsity": 79.93344907407408,
                    "linear_total": 84934656,
                    "nnz": 40951962,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "3": [
                            2,
                            3,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            8,
                            9,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            6,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108875714,
                    "total_sparsity": 62.38650430342987
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl40_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 4,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl40_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_abr32_abc32_it0_fw10_r-l1_rfl40_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.69392512957342,
            "fill_rate": 0.2006655092592592,
            "speedup": 2.799991523936488
        }
    ],
    "Block/struct method, final fine tuned, s=b, t=l": [
        {
            "annotate": "23",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            10,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 81.16367076631977,
                    "f1": 88.30880074775172
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad4___dpm-sigmoied_threshold:1d_alt_ap--17cd29ad8a563746/checkpoint-47500",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test4/hp_od-__data_2to__devel_data__nn_pruning__output__squad4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad4___dpm-sigmoied_threshold:1d_alt_ap--17cd29ad8a563746/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 16.733935527801513,
                    "eval_elapsed_time": 23.897289094515145
                },
                "speedup": 2.306354828560857,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 310272,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1293312,
                            "linear_total": 7077888,
                            "nnz": 1299082,
                            "total": 7086528
                        },
                        "1": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 436224,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1419264,
                            "linear_total": 7077888,
                            "nnz": 1425116,
                            "total": 7086528
                        },
                        "10": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 121344,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1104384,
                            "linear_total": 7077888,
                            "nnz": 1110031,
                            "total": 7086528
                        },
                        "11": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 156672,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1139712,
                            "linear_total": 7077888,
                            "nnz": 1145382,
                            "total": 7086528
                        },
                        "2": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 543744,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2313216,
                            "linear_total": 7077888,
                            "nnz": 2319906,
                            "total": 7087296
                        },
                        "3": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 565248,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2138112,
                            "linear_total": 7077888,
                            "nnz": 2144624,
                            "total": 7087104
                        },
                        "4": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 589824,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2359296,
                            "linear_total": 7077888,
                            "nnz": 2366016,
                            "total": 7087296
                        },
                        "5": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 514560,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1694208,
                            "linear_total": 7077888,
                            "nnz": 1700303,
                            "total": 7086720
                        },
                        "6": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 442368,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2015232,
                            "linear_total": 7077888,
                            "nnz": 2021664,
                            "total": 7087104
                        },
                        "7": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 322560,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1698816,
                            "linear_total": 7077888,
                            "nnz": 1704978,
                            "total": 7086912
                        },
                        "8": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 167424,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1740288,
                            "linear_total": 7077888,
                            "nnz": 1746541,
                            "total": 7087104
                        },
                        "9": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 84480,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1264128,
                            "linear_total": 7077888,
                            "nnz": 1269943,
                            "total": 7086720
                        }
                    },
                    "linear_nnz": 20179968,
                    "linear_sparsity": 76.24059606481481,
                    "linear_total": 84934656,
                    "nnz": 44092308,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            10,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108881090,
                    "total_sparsity": 59.50416367066127
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad4___dpm-sigmoied_threshold:1d_alt_ap--17cd29ad8a563746",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad4___dpm-sigmoied_threshold:1d_alt_ap--17cd29ad8a563746",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad4___dpm-sigmoied_threshold:1d_alt_ap--17cd29ad8a563746",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.30880074775172,
            "fill_rate": 0.23759403935185186,
            "speedup": 2.306354828560857
        },
        {
            "annotate": "23",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            10,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 81.01229895931883,
                    "f1": 88.25975710073095
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad4___dpm-sigmoied_threshold:1d_alt_ap--17cd29ad8a563746/checkpoint-52500",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test4/hp_od-__data_2to__devel_data__nn_pruning__output__squad4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad4___dpm-sigmoied_threshold:1d_alt_ap--17cd29ad8a563746/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 16.33017548751831,
                    "eval_elapsed_time": 23.592021488584578
                },
                "speedup": 2.363378950512201,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 310272,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1293312,
                            "linear_total": 7077888,
                            "nnz": 1299082,
                            "total": 7086528
                        },
                        "1": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 436224,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1419264,
                            "linear_total": 7077888,
                            "nnz": 1425116,
                            "total": 7086528
                        },
                        "10": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 121344,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1104384,
                            "linear_total": 7077888,
                            "nnz": 1110031,
                            "total": 7086528
                        },
                        "11": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 156672,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1139712,
                            "linear_total": 7077888,
                            "nnz": 1145382,
                            "total": 7086528
                        },
                        "2": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 543744,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2313216,
                            "linear_total": 7077888,
                            "nnz": 2319906,
                            "total": 7087296
                        },
                        "3": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 565248,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2138112,
                            "linear_total": 7077888,
                            "nnz": 2144624,
                            "total": 7087104
                        },
                        "4": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 589824,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2359296,
                            "linear_total": 7077888,
                            "nnz": 2366016,
                            "total": 7087296
                        },
                        "5": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 514560,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1694208,
                            "linear_total": 7077888,
                            "nnz": 1700303,
                            "total": 7086720
                        },
                        "6": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 442368,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2015232,
                            "linear_total": 7077888,
                            "nnz": 2021664,
                            "total": 7087104
                        },
                        "7": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 322560,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1698816,
                            "linear_total": 7077888,
                            "nnz": 1704978,
                            "total": 7086912
                        },
                        "8": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 167424,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1740288,
                            "linear_total": 7077888,
                            "nnz": 1746541,
                            "total": 7087104
                        },
                        "9": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 84480,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1264128,
                            "linear_total": 7077888,
                            "nnz": 1269943,
                            "total": 7086720
                        }
                    },
                    "linear_nnz": 20179968,
                    "linear_sparsity": 76.24059606481481,
                    "linear_total": 84934656,
                    "nnz": 44092308,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            10,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108881090,
                    "total_sparsity": 59.50416367066127
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad4___dpm-sigmoied_threshold:1d_alt_ap--17cd29ad8a563746",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad4___dpm-sigmoied_threshold:1d_alt_ap--17cd29ad8a563746",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad4___dpm-sigmoied_threshold:1d_alt_ap--17cd29ad8a563746",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.25975710073095,
            "fill_rate": 0.23759403935185186,
            "speedup": 2.363378950512201
        },
        {
            "annotate": "23",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            10,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.94607379375591,
                    "f1": 88.20838283474134
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad4___dpm-sigmoied_threshold:1d_alt_ap--17cd29ad8a563746/checkpoint-55000",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test4/hp_od-__data_2to__devel_data__nn_pruning__output__squad4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad4___dpm-sigmoied_threshold:1d_alt_ap--17cd29ad8a563746/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 16.322325397491454,
                    "eval_elapsed_time": 23.561688547953963
                },
                "speedup": 2.36451559845159,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 310272,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1293312,
                            "linear_total": 7077888,
                            "nnz": 1299082,
                            "total": 7086528
                        },
                        "1": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 436224,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1419264,
                            "linear_total": 7077888,
                            "nnz": 1425116,
                            "total": 7086528
                        },
                        "10": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 121344,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1104384,
                            "linear_total": 7077888,
                            "nnz": 1110031,
                            "total": 7086528
                        },
                        "11": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 156672,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1139712,
                            "linear_total": 7077888,
                            "nnz": 1145382,
                            "total": 7086528
                        },
                        "2": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 543744,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2313216,
                            "linear_total": 7077888,
                            "nnz": 2319906,
                            "total": 7087296
                        },
                        "3": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 565248,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2138112,
                            "linear_total": 7077888,
                            "nnz": 2144624,
                            "total": 7087104
                        },
                        "4": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 589824,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2359296,
                            "linear_total": 7077888,
                            "nnz": 2366016,
                            "total": 7087296
                        },
                        "5": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 514560,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1694208,
                            "linear_total": 7077888,
                            "nnz": 1700303,
                            "total": 7086720
                        },
                        "6": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 442368,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2015232,
                            "linear_total": 7077888,
                            "nnz": 2021664,
                            "total": 7087104
                        },
                        "7": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 322560,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1698816,
                            "linear_total": 7077888,
                            "nnz": 1704978,
                            "total": 7086912
                        },
                        "8": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 167424,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1740288,
                            "linear_total": 7077888,
                            "nnz": 1746541,
                            "total": 7087104
                        },
                        "9": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 84480,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1264128,
                            "linear_total": 7077888,
                            "nnz": 1269943,
                            "total": 7086720
                        }
                    },
                    "linear_nnz": 20179968,
                    "linear_sparsity": 76.24059606481481,
                    "linear_total": 84934656,
                    "nnz": 44092308,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            10,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108881090,
                    "total_sparsity": 59.50416367066127
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad4___dpm-sigmoied_threshold:1d_alt_ap--17cd29ad8a563746",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad4___dpm-sigmoied_threshold:1d_alt_ap--17cd29ad8a563746",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad4___dpm-sigmoied_threshold:1d_alt_ap--17cd29ad8a563746",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.20838283474134,
            "fill_rate": 0.23759403935185186,
            "speedup": 2.36451559845159
        },
        {
            "annotate": "23",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            10,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.91769157994324,
                    "f1": 88.19995861416005
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad4___dpm-sigmoied_threshold:1d_alt_ap--17cd29ad8a563746/checkpoint-55330",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test4/hp_od-__data_2to__devel_data__nn_pruning__output__squad4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad4___dpm-sigmoied_threshold:1d_alt_ap--17cd29ad8a563746/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 16.33160983657837,
                    "eval_elapsed_time": 23.633546161465347
                },
                "speedup": 2.363171383076403,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 310272,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1293312,
                            "linear_total": 7077888,
                            "nnz": 1299082,
                            "total": 7086528
                        },
                        "1": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 436224,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1419264,
                            "linear_total": 7077888,
                            "nnz": 1425116,
                            "total": 7086528
                        },
                        "10": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 121344,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1104384,
                            "linear_total": 7077888,
                            "nnz": 1110031,
                            "total": 7086528
                        },
                        "11": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 156672,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1139712,
                            "linear_total": 7077888,
                            "nnz": 1145382,
                            "total": 7086528
                        },
                        "2": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 543744,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2313216,
                            "linear_total": 7077888,
                            "nnz": 2319906,
                            "total": 7087296
                        },
                        "3": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 565248,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2138112,
                            "linear_total": 7077888,
                            "nnz": 2144624,
                            "total": 7087104
                        },
                        "4": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 589824,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2359296,
                            "linear_total": 7077888,
                            "nnz": 2366016,
                            "total": 7087296
                        },
                        "5": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 514560,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1694208,
                            "linear_total": 7077888,
                            "nnz": 1700303,
                            "total": 7086720
                        },
                        "6": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 442368,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2015232,
                            "linear_total": 7077888,
                            "nnz": 2021664,
                            "total": 7087104
                        },
                        "7": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 322560,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1698816,
                            "linear_total": 7077888,
                            "nnz": 1704978,
                            "total": 7086912
                        },
                        "8": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 167424,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1740288,
                            "linear_total": 7077888,
                            "nnz": 1746541,
                            "total": 7087104
                        },
                        "9": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 84480,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1264128,
                            "linear_total": 7077888,
                            "nnz": 1269943,
                            "total": 7086720
                        }
                    },
                    "linear_nnz": 20179968,
                    "linear_sparsity": 76.24059606481481,
                    "linear_total": 84934656,
                    "nnz": 44092308,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            10,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108881090,
                    "total_sparsity": 59.50416367066127
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad4___dpm-sigmoied_threshold:1d_alt_ap--17cd29ad8a563746",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad4___dpm-sigmoied_threshold:1d_alt_ap--17cd29ad8a563746",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad4___dpm-sigmoied_threshold:1d_alt_ap--17cd29ad8a563746",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.19995861416005,
            "fill_rate": 0.23759403935185186,
            "speedup": 2.363171383076403
        },
        {
            "annotate": "29",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            2,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            1,
                            2
                        ],
                        "5": [
                            1,
                            2,
                            6
                        ],
                        "6": [
                            2,
                            3
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 81.97729422894986,
                    "f1": 89.00029318511001
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad4___dpm-sigmoied_threshold:1d_alt_ap--754f92d6579864ca/checkpoint-52500",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test4/hp_od-__data_2to__devel_data__nn_pruning__output__squad4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad4___dpm-sigmoied_threshold:1d_alt_ap--754f92d6579864ca/checkpoint-95000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 18.596232307434082,
                    "eval_elapsed_time": 25.85930221248418
                },
                "speedup": 2.075387764969063,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 543744,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1920000,
                            "linear_total": 7077888,
                            "nnz": 1926306,
                            "total": 7086912
                        },
                        "1": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 764928,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1747968,
                            "linear_total": 7077888,
                            "nnz": 1754034,
                            "total": 7086528
                        },
                        "10": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 176640,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1356288,
                            "linear_total": 7077888,
                            "nnz": 1362163,
                            "total": 7086720
                        },
                        "11": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 259584,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1439232,
                            "linear_total": 7077888,
                            "nnz": 1445161,
                            "total": 7086720
                        },
                        "2": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 835584,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2605056,
                            "linear_total": 7077888,
                            "nnz": 2611936,
                            "total": 7087296
                        },
                        "3": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 866304,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2635776,
                            "linear_total": 7077888,
                            "nnz": 2642676,
                            "total": 7087296
                        },
                        "4": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 893952,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2860032,
                            "linear_total": 7077888,
                            "nnz": 2867142,
                            "total": 7087488
                        },
                        "5": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 797184,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2566656,
                            "linear_total": 7077888,
                            "nnz": 2573511,
                            "total": 7087296
                        },
                        "6": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 637440,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2603520,
                            "linear_total": 7077888,
                            "nnz": 2610463,
                            "total": 7087488
                        },
                        "7": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 473088,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1849344,
                            "linear_total": 7077888,
                            "nnz": 1855604,
                            "total": 7086912
                        },
                        "8": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 241152,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1814016,
                            "linear_total": 7077888,
                            "nnz": 1820317,
                            "total": 7087104
                        },
                        "9": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 119808,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1299456,
                            "linear_total": 7077888,
                            "nnz": 1305294,
                            "total": 7086720
                        }
                    },
                    "linear_nnz": 24697344,
                    "linear_sparsity": 70.92194733796296,
                    "linear_total": 84934656,
                    "nnz": 48613329,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            2,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            1,
                            2
                        ],
                        "5": [
                            1,
                            2,
                            6
                        ],
                        "6": [
                            2,
                            3
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108883202,
                    "total_sparsity": 55.352774250705814
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad4___dpm-sigmoied_threshold:1d_alt_ap--754f92d6579864ca",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad4___dpm-sigmoied_threshold:1d_alt_ap--754f92d6579864ca",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad4___dpm-sigmoied_threshold:1d_alt_ap--754f92d6579864ca",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 89.00029318511001,
            "fill_rate": 0.29078052662037035,
            "speedup": 2.075387764969063
        },
        {
            "annotate": "29",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            2,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            1,
                            2
                        ],
                        "5": [
                            1,
                            2,
                            6
                        ],
                        "6": [
                            2,
                            3
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 81.9205298013245,
                    "f1": 88.96937906487014
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad4___dpm-sigmoied_threshold:1d_alt_ap--754f92d6579864ca/checkpoint-55330",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test4/hp_od-__data_2to__devel_data__nn_pruning__output__squad4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad4___dpm-sigmoied_threshold:1d_alt_ap--754f92d6579864ca/checkpoint-95000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 18.60489262008667,
                    "eval_elapsed_time": 25.903281938284636
                },
                "speedup": 2.0744217015100035,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 543744,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1920000,
                            "linear_total": 7077888,
                            "nnz": 1926306,
                            "total": 7086912
                        },
                        "1": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 764928,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1747968,
                            "linear_total": 7077888,
                            "nnz": 1754034,
                            "total": 7086528
                        },
                        "10": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 176640,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1356288,
                            "linear_total": 7077888,
                            "nnz": 1362163,
                            "total": 7086720
                        },
                        "11": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 259584,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1439232,
                            "linear_total": 7077888,
                            "nnz": 1445161,
                            "total": 7086720
                        },
                        "2": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 835584,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2605056,
                            "linear_total": 7077888,
                            "nnz": 2611936,
                            "total": 7087296
                        },
                        "3": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 866304,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2635776,
                            "linear_total": 7077888,
                            "nnz": 2642676,
                            "total": 7087296
                        },
                        "4": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 893952,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2860032,
                            "linear_total": 7077888,
                            "nnz": 2867142,
                            "total": 7087488
                        },
                        "5": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 797184,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2566656,
                            "linear_total": 7077888,
                            "nnz": 2573511,
                            "total": 7087296
                        },
                        "6": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 637440,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2603520,
                            "linear_total": 7077888,
                            "nnz": 2610463,
                            "total": 7087488
                        },
                        "7": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 473088,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1849344,
                            "linear_total": 7077888,
                            "nnz": 1855604,
                            "total": 7086912
                        },
                        "8": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 241152,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1814016,
                            "linear_total": 7077888,
                            "nnz": 1820317,
                            "total": 7087104
                        },
                        "9": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 119808,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1299456,
                            "linear_total": 7077888,
                            "nnz": 1305294,
                            "total": 7086720
                        }
                    },
                    "linear_nnz": 24697344,
                    "linear_sparsity": 70.92194733796296,
                    "linear_total": 84934656,
                    "nnz": 48613329,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            2,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            1,
                            2
                        ],
                        "5": [
                            1,
                            2,
                            6
                        ],
                        "6": [
                            2,
                            3
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108883202,
                    "total_sparsity": 55.352774250705814
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad4___dpm-sigmoied_threshold:1d_alt_ap--754f92d6579864ca",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad4___dpm-sigmoied_threshold:1d_alt_ap--754f92d6579864ca",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad4___dpm-sigmoied_threshold:1d_alt_ap--754f92d6579864ca",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.96937906487014,
            "fill_rate": 0.29078052662037035,
            "speedup": 2.0744217015100035
        },
        {
            "annotate": "19",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            1,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            1,
                            2,
                            6,
                            7,
                            8,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "9": [
                            1,
                            2,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.48912015137181,
                    "f1": 87.15966661496869
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold--696f4785b3ba52e7/checkpoint-55330",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test4/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold--696f4785b3ba52e7/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 14.738462112426758,
                    "eval_elapsed_time": 21.50075929518789
                },
                "speedup": 2.6186173775098402,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 222720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1009152,
                            "linear_total": 7077888,
                            "nnz": 1014673,
                            "total": 7086336
                        },
                        "1": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 344064,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1130496,
                            "linear_total": 7077888,
                            "nnz": 1136096,
                            "total": 7086336
                        },
                        "10": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 104448,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1087488,
                            "linear_total": 7077888,
                            "nnz": 1093124,
                            "total": 7086528
                        },
                        "11": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 129024,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 915456,
                            "linear_total": 7077888,
                            "nnz": 920916,
                            "total": 7086336
                        },
                        "2": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 437760,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2010624,
                            "linear_total": 7077888,
                            "nnz": 2017053,
                            "total": 7087104
                        },
                        "3": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 440832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1817088,
                            "linear_total": 7077888,
                            "nnz": 1823327,
                            "total": 7086912
                        },
                        "4": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 443904,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1623552,
                            "linear_total": 7077888,
                            "nnz": 1629601,
                            "total": 7086720
                        },
                        "5": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 427008,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1410048,
                            "linear_total": 7077888,
                            "nnz": 1415894,
                            "total": 7086528
                        },
                        "6": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 334848,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1711104,
                            "linear_total": 7077888,
                            "nnz": 1717274,
                            "total": 7086912
                        },
                        "7": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 265728,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1641984,
                            "linear_total": 7077888,
                            "nnz": 1648109,
                            "total": 7086912
                        },
                        "8": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 138240,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1317888,
                            "linear_total": 7077888,
                            "nnz": 1323738,
                            "total": 7086720
                        },
                        "9": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 67584,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1050624,
                            "linear_total": 7077888,
                            "nnz": 1056236,
                            "total": 7086528
                        }
                    },
                    "linear_nnz": 16725504,
                    "linear_sparsity": 80.3077980324074,
                    "linear_total": 84934656,
                    "nnz": 40634763,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            1,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            1,
                            2,
                            6,
                            7,
                            8,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "9": [
                            1,
                            2,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108878594,
                    "total_sparsity": 62.67883198418232
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold--696f4785b3ba52e7",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold--696f4785b3ba52e7",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold--696f4785b3ba52e7",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.15966661496869,
            "fill_rate": 0.19692201967592593,
            "speedup": 2.6186173775098402
        },
        {
            "annotate": "35",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0,
                            4,
                            5,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7
                        ],
                        "11": [
                            8,
                            10,
                            5,
                            7
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            1
                        ],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            2,
                            3
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 82.95175023651845,
                    "f1": 89.81145528969563
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold--6b3d26fc7262a898/checkpoint-47500",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test4/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold--6b3d26fc7262a898/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 21.051361206054686,
                    "eval_elapsed_time": 28.537553551606834
                },
                "speedup": 1.8333442967227587,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 964608,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2537472,
                            "linear_total": 7077888,
                            "nnz": 2544244,
                            "total": 7087104
                        },
                        "1": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1202688,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2382336,
                            "linear_total": 7077888,
                            "nnz": 2388879,
                            "total": 7086720
                        },
                        "10": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 256512,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1632768,
                            "linear_total": 7077888,
                            "nnz": 1638887,
                            "total": 7086912
                        },
                        "11": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 400896,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1973760,
                            "linear_total": 7077888,
                            "nnz": 1980165,
                            "total": 7087104
                        },
                        "2": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1281024,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3247104,
                            "linear_total": 7077888,
                            "nnz": 3254466,
                            "total": 7087488
                        },
                        "3": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1319424,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3088896,
                            "linear_total": 7077888,
                            "nnz": 3096091,
                            "total": 7087296
                        },
                        "4": {
                            "linear_attention_nnz": 2162688,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1288704,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3451392,
                            "linear_total": 7077888,
                            "nnz": 3458951,
                            "total": 7087680
                        },
                        "5": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1210368,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3176448,
                            "linear_total": 7077888,
                            "nnz": 3183764,
                            "total": 7087488
                        },
                        "6": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 940032,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2906112,
                            "linear_total": 7077888,
                            "nnz": 2913252,
                            "total": 7087488
                        },
                        "7": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 662016,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2234880,
                            "linear_total": 7077888,
                            "nnz": 2241455,
                            "total": 7087104
                        },
                        "8": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 348672,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2118144,
                            "linear_total": 7077888,
                            "nnz": 2124707,
                            "total": 7087296
                        },
                        "9": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 159744,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1536000,
                            "linear_total": 7077888,
                            "nnz": 1542056,
                            "total": 7086912
                        }
                    },
                    "linear_nnz": 30285312,
                    "linear_sparsity": 64.3428096064815,
                    "linear_total": 84934656,
                    "nnz": 54205639,
                    "pruned_heads": {
                        "0": [
                            0,
                            4,
                            5,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7
                        ],
                        "11": [
                            8,
                            10,
                            5,
                            7
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            1
                        ],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            2,
                            3
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "total": 108885314,
                    "total_sparsity": 50.21767673829732
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold--6b3d26fc7262a898",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold--6b3d26fc7262a898",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold--6b3d26fc7262a898",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 89.81145528969563,
            "fill_rate": 0.356571903935185,
            "speedup": 1.8333442967227587
        },
        {
            "annotate": "35",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0,
                            4,
                            5,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7
                        ],
                        "11": [
                            8,
                            10,
                            5,
                            7
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            1
                        ],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            2,
                            3
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 83.01797540208136,
                    "f1": 89.80879102775579
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold--6b3d26fc7262a898/checkpoint-55330",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test4/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold--6b3d26fc7262a898/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 21.06306128692627,
                    "eval_elapsed_time": 28.405260108411312
                },
                "speedup": 1.8323259131058234,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 964608,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2537472,
                            "linear_total": 7077888,
                            "nnz": 2544244,
                            "total": 7087104
                        },
                        "1": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1202688,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2382336,
                            "linear_total": 7077888,
                            "nnz": 2388879,
                            "total": 7086720
                        },
                        "10": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 256512,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1632768,
                            "linear_total": 7077888,
                            "nnz": 1638887,
                            "total": 7086912
                        },
                        "11": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 400896,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1973760,
                            "linear_total": 7077888,
                            "nnz": 1980165,
                            "total": 7087104
                        },
                        "2": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1281024,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3247104,
                            "linear_total": 7077888,
                            "nnz": 3254466,
                            "total": 7087488
                        },
                        "3": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1319424,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3088896,
                            "linear_total": 7077888,
                            "nnz": 3096091,
                            "total": 7087296
                        },
                        "4": {
                            "linear_attention_nnz": 2162688,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1288704,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3451392,
                            "linear_total": 7077888,
                            "nnz": 3458951,
                            "total": 7087680
                        },
                        "5": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1210368,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3176448,
                            "linear_total": 7077888,
                            "nnz": 3183764,
                            "total": 7087488
                        },
                        "6": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 940032,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2906112,
                            "linear_total": 7077888,
                            "nnz": 2913252,
                            "total": 7087488
                        },
                        "7": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 662016,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2234880,
                            "linear_total": 7077888,
                            "nnz": 2241455,
                            "total": 7087104
                        },
                        "8": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 348672,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2118144,
                            "linear_total": 7077888,
                            "nnz": 2124707,
                            "total": 7087296
                        },
                        "9": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 159744,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1536000,
                            "linear_total": 7077888,
                            "nnz": 1542056,
                            "total": 7086912
                        }
                    },
                    "linear_nnz": 30285312,
                    "linear_sparsity": 64.3428096064815,
                    "linear_total": 84934656,
                    "nnz": 54205639,
                    "pruned_heads": {
                        "0": [
                            0,
                            4,
                            5,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7
                        ],
                        "11": [
                            8,
                            10,
                            5,
                            7
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            1
                        ],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            2,
                            3
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "total": 108885314,
                    "total_sparsity": 50.21767673829732
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold--6b3d26fc7262a898",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold--6b3d26fc7262a898",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold--6b3d26fc7262a898",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 89.80879102775579,
            "fill_rate": 0.356571903935185,
            "speedup": 1.8323259131058234
        },
        {
            "annotate": "45",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0
                        ],
                        "1": [
                            0,
                            2,
                            3
                        ],
                        "10": [
                            1,
                            4,
                            7
                        ],
                        "11": [
                            8,
                            5,
                            7
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            2,
                            4
                        ],
                        "4": [],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            3
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [],
                        "9": [
                            1,
                            4,
                            5,
                            7
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 83.72753074739829,
                    "f1": 90.26969803251558
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold:--d169c0ebde721c7/checkpoint-55000",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test4/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold:--d169c0ebde721c7/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 24.729826789855956,
                    "eval_elapsed_time": 31.979490760713816
                },
                "speedup": 1.5606414607482133,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 2162688,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1720320,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3883008,
                            "linear_total": 7077888,
                            "nnz": 3890848,
                            "total": 7087680
                        },
                        "1": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1815552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3585024,
                            "linear_total": 7077888,
                            "nnz": 3592542,
                            "total": 7087296
                        },
                        "10": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 391680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2161152,
                            "linear_total": 7077888,
                            "nnz": 2167743,
                            "total": 7087296
                        },
                        "11": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 638976,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2408448,
                            "linear_total": 7077888,
                            "nnz": 2415200,
                            "total": 7087296
                        },
                        "2": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1961472,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3927552,
                            "linear_total": 7077888,
                            "nnz": 3935357,
                            "total": 7087488
                        },
                        "3": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1981440,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3947520,
                            "linear_total": 7077888,
                            "nnz": 3955338,
                            "total": 7087488
                        },
                        "4": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1850880,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4210176,
                            "linear_total": 7077888,
                            "nnz": 4218293,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1752576,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3718656,
                            "linear_total": 7077888,
                            "nnz": 3726325,
                            "total": 7087488
                        },
                        "6": {
                            "linear_attention_nnz": 2162688,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1350144,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3512832,
                            "linear_total": 7077888,
                            "nnz": 3520431,
                            "total": 7087680
                        },
                        "7": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1013760,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2586624,
                            "linear_total": 7077888,
                            "nnz": 2593428,
                            "total": 7087104
                        },
                        "8": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 511488,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2870784,
                            "linear_total": 7077888,
                            "nnz": 2878029,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 254976,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1827840,
                            "linear_total": 7077888,
                            "nnz": 1834150,
                            "total": 7087104
                        }
                    },
                    "linear_nnz": 38639616,
                    "linear_sparsity": 54.506655092592595,
                    "linear_total": 84934656,
                    "nnz": 62566406,
                    "pruned_heads": {
                        "0": [
                            0
                        ],
                        "1": [
                            0,
                            2,
                            3
                        ],
                        "10": [
                            1,
                            4,
                            7
                        ],
                        "11": [
                            8,
                            5,
                            7
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            2,
                            4
                        ],
                        "4": [],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            3
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [],
                        "9": [
                            1,
                            4,
                            5,
                            7
                        ]
                    },
                    "total": 108888386,
                    "total_sparsity": 42.54079034654807
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold:--d169c0ebde721c7",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold:--d169c0ebde721c7",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold:--d169c0ebde721c7",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 90.26969803251558,
            "fill_rate": 0.45493344907407407,
            "speedup": 1.5606414607482133
        },
        {
            "annotate": "45",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0
                        ],
                        "1": [
                            0,
                            2,
                            3
                        ],
                        "10": [
                            1,
                            4,
                            7
                        ],
                        "11": [
                            8,
                            5,
                            7
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            2,
                            4
                        ],
                        "4": [],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            3
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [],
                        "9": [
                            1,
                            4,
                            5,
                            7
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 83.71807000946073,
                    "f1": 90.26096311838805
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold:--d169c0ebde721c7/checkpoint-55330",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test4/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold:--d169c0ebde721c7/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 24.76904414367676,
                    "eval_elapsed_time": 32.039006903767586
                },
                "speedup": 1.5581704639669667,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 2162688,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1720320,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3883008,
                            "linear_total": 7077888,
                            "nnz": 3890848,
                            "total": 7087680
                        },
                        "1": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1815552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3585024,
                            "linear_total": 7077888,
                            "nnz": 3592542,
                            "total": 7087296
                        },
                        "10": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 391680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2161152,
                            "linear_total": 7077888,
                            "nnz": 2167743,
                            "total": 7087296
                        },
                        "11": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 638976,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2408448,
                            "linear_total": 7077888,
                            "nnz": 2415200,
                            "total": 7087296
                        },
                        "2": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1961472,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3927552,
                            "linear_total": 7077888,
                            "nnz": 3935357,
                            "total": 7087488
                        },
                        "3": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1981440,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3947520,
                            "linear_total": 7077888,
                            "nnz": 3955338,
                            "total": 7087488
                        },
                        "4": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1850880,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4210176,
                            "linear_total": 7077888,
                            "nnz": 4218293,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1752576,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3718656,
                            "linear_total": 7077888,
                            "nnz": 3726325,
                            "total": 7087488
                        },
                        "6": {
                            "linear_attention_nnz": 2162688,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1350144,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3512832,
                            "linear_total": 7077888,
                            "nnz": 3520431,
                            "total": 7087680
                        },
                        "7": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1013760,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2586624,
                            "linear_total": 7077888,
                            "nnz": 2593428,
                            "total": 7087104
                        },
                        "8": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 511488,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2870784,
                            "linear_total": 7077888,
                            "nnz": 2878029,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 254976,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1827840,
                            "linear_total": 7077888,
                            "nnz": 1834150,
                            "total": 7087104
                        }
                    },
                    "linear_nnz": 38639616,
                    "linear_sparsity": 54.506655092592595,
                    "linear_total": 84934656,
                    "nnz": 62566406,
                    "pruned_heads": {
                        "0": [
                            0
                        ],
                        "1": [
                            0,
                            2,
                            3
                        ],
                        "10": [
                            1,
                            4,
                            7
                        ],
                        "11": [
                            8,
                            5,
                            7
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            2,
                            4
                        ],
                        "4": [],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            3
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [],
                        "9": [
                            1,
                            4,
                            5,
                            7
                        ]
                    },
                    "total": 108888386,
                    "total_sparsity": 42.54079034654807
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold:--d169c0ebde721c7",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold:--d169c0ebde721c7",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold:--d169c0ebde721c7",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 90.26096311838805,
            "fill_rate": 0.45493344907407407,
            "speedup": 1.5581704639669667
        }
    ],
    "Block/struct method, final fine tuned, s=l": [
        {
            "annotate": "17",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 1024,
                    "initializer_range": 0.02,
                    "intermediate_size": 4096,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 16,
                    "num_hidden_layers": 24,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            2,
                            3,
                            4,
                            7,
                            8,
                            9,
                            10,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            7,
                            8,
                            10,
                            12,
                            13,
                            14
                        ],
                        "11": [
                            0,
                            2,
                            4,
                            5,
                            8,
                            10
                        ],
                        "12": [
                            10,
                            3,
                            13,
                            6
                        ],
                        "13": [
                            2,
                            10,
                            4,
                            12
                        ],
                        "14": [
                            2,
                            3,
                            4,
                            8,
                            11
                        ],
                        "15": [
                            0,
                            5,
                            6,
                            7,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            8,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            11,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            11,
                            13
                        ],
                        "19": [
                            0,
                            2,
                            3,
                            4,
                            9,
                            10,
                            11,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "21": [
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "23": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            13,
                            14
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            12,
                            13,
                            15
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 83.74645222327341,
                    "f1": 90.16320537561052
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_10/checkpoint-47500",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test_large/large_regu_10/checkpoint-215000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 37.53850735473633,
                    "eval_elapsed_time": 44.58338421070948
                },
                "speedup": 1.0281280670181348,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 192512,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1765376,
                            "linear_total": 12582912,
                            "nnz": 1772766,
                            "total": 12594304
                        },
                        "1": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 270336,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 794624,
                            "linear_total": 12582912,
                            "nnz": 801284,
                            "total": 12593536
                        },
                        "10": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 995328,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3616768,
                            "linear_total": 12582912,
                            "nnz": 3625318,
                            "total": 12595072
                        },
                        "11": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1032192,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3653632,
                            "linear_total": 12582912,
                            "nnz": 3662200,
                            "total": 12595072
                        },
                        "12": {
                            "linear_attention_nnz": 3145728,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1241088,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4386816,
                            "linear_total": 12582912,
                            "nnz": 4395870,
                            "total": 12595456
                        },
                        "13": {
                            "linear_attention_nnz": 3145728,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1179648,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4325376,
                            "linear_total": 12582912,
                            "nnz": 4334400,
                            "total": 12595456
                        },
                        "14": {
                            "linear_attention_nnz": 2883584,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 909312,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3792896,
                            "linear_total": 12582912,
                            "nnz": 3801596,
                            "total": 12595264
                        },
                        "15": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 681984,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3303424,
                            "linear_total": 12582912,
                            "nnz": 3311821,
                            "total": 12595072
                        },
                        "16": {
                            "linear_attention_nnz": 2883584,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 473088,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3356672,
                            "linear_total": 12582912,
                            "nnz": 3365159,
                            "total": 12595264
                        },
                        "17": {
                            "linear_attention_nnz": 2883584,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 368640,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3252224,
                            "linear_total": 12582912,
                            "nnz": 3260660,
                            "total": 12595264
                        },
                        "18": {
                            "linear_attention_nnz": 2883584,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 321536,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3205120,
                            "linear_total": 12582912,
                            "nnz": 3213533,
                            "total": 12595264
                        },
                        "19": {
                            "linear_attention_nnz": 2097152,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 270336,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2367488,
                            "linear_total": 12582912,
                            "nnz": 2375300,
                            "total": 12594688
                        },
                        "2": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 286720,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 811008,
                            "linear_total": 12582912,
                            "nnz": 817676,
                            "total": 12593536
                        },
                        "20": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 112640,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 899072,
                            "linear_total": 12582912,
                            "nnz": 905847,
                            "total": 12593728
                        },
                        "21": {
                            "linear_attention_nnz": 1310720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 77824,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1388544,
                            "linear_total": 12582912,
                            "nnz": 1395686,
                            "total": 12594112
                        },
                        "22": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 79872,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 866304,
                            "linear_total": 12582912,
                            "nnz": 873063,
                            "total": 12593728
                        },
                        "23": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 182272,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1230848,
                            "linear_total": 12582912,
                            "nnz": 1237849,
                            "total": 12593920
                        },
                        "3": {
                            "linear_attention_nnz": 1310720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 413696,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1724416,
                            "linear_total": 12582912,
                            "nnz": 1731722,
                            "total": 12594112
                        },
                        "4": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 466944,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 991232,
                            "linear_total": 12582912,
                            "nnz": 997988,
                            "total": 12593536
                        },
                        "5": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 552960,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1077248,
                            "linear_total": 12582912,
                            "nnz": 1084046,
                            "total": 12593536
                        },
                        "6": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 608256,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1394688,
                            "linear_total": 12582912,
                            "nnz": 1401705,
                            "total": 12593728
                        },
                        "7": {
                            "linear_attention_nnz": 1310720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 438272,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1748992,
                            "linear_total": 12582912,
                            "nnz": 1756310,
                            "total": 12594112
                        },
                        "8": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 661504,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1710080,
                            "linear_total": 12582912,
                            "nnz": 1717315,
                            "total": 12593920
                        },
                        "9": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 747520,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2320384,
                            "linear_total": 12582912,
                            "nnz": 2328045,
                            "total": 12594304
                        }
                    },
                    "linear_nnz": 53983232,
                    "linear_sparsity": 82.12415907118056,
                    "linear_total": 301989888,
                    "nnz": 85952121,
                    "pruned_heads": {
                        "0": [
                            2,
                            3,
                            4,
                            7,
                            8,
                            9,
                            10,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            7,
                            8,
                            10,
                            12,
                            13,
                            14
                        ],
                        "11": [
                            0,
                            2,
                            4,
                            5,
                            8,
                            10
                        ],
                        "12": [
                            10,
                            3,
                            13,
                            6
                        ],
                        "13": [
                            2,
                            10,
                            4,
                            12
                        ],
                        "14": [
                            2,
                            3,
                            4,
                            8,
                            11
                        ],
                        "15": [
                            0,
                            5,
                            6,
                            7,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            8,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            11,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            11,
                            13
                        ],
                        "19": [
                            0,
                            2,
                            3,
                            4,
                            9,
                            10,
                            11,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "21": [
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "23": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            13,
                            14
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            12,
                            13,
                            15
                        ]
                    },
                    "total": 334050946,
                    "total_sparsity": 74.26975674542769
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_10",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_10",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_10",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 90.16320537561052,
            "fill_rate": 0.4766890914351851,
            "speedup": 1.0281280670181348
        },
        {
            "annotate": "17",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 1024,
                    "initializer_range": 0.02,
                    "intermediate_size": 4096,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 16,
                    "num_hidden_layers": 24,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            2,
                            3,
                            4,
                            7,
                            8,
                            9,
                            10,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            7,
                            8,
                            10,
                            12,
                            13,
                            14
                        ],
                        "11": [
                            0,
                            2,
                            4,
                            5,
                            8,
                            10
                        ],
                        "12": [
                            10,
                            3,
                            13,
                            6
                        ],
                        "13": [
                            2,
                            10,
                            4,
                            12
                        ],
                        "14": [
                            2,
                            3,
                            4,
                            8,
                            11
                        ],
                        "15": [
                            0,
                            5,
                            6,
                            7,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            8,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            11,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            11,
                            13
                        ],
                        "19": [
                            0,
                            2,
                            3,
                            4,
                            9,
                            10,
                            11,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "21": [
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "23": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            13,
                            14
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            12,
                            13,
                            15
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 83.62346263008514,
                    "f1": 90.10843526218638
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_10/checkpoint-55330",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test_large/large_regu_10/checkpoint-215000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 37.30008307647705,
                    "eval_elapsed_time": 44.469506811816245
                },
                "speedup": 1.034699920808227,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 192512,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1765376,
                            "linear_total": 12582912,
                            "nnz": 1772766,
                            "total": 12594304
                        },
                        "1": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 270336,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 794624,
                            "linear_total": 12582912,
                            "nnz": 801284,
                            "total": 12593536
                        },
                        "10": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 995328,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3616768,
                            "linear_total": 12582912,
                            "nnz": 3625318,
                            "total": 12595072
                        },
                        "11": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1032192,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3653632,
                            "linear_total": 12582912,
                            "nnz": 3662200,
                            "total": 12595072
                        },
                        "12": {
                            "linear_attention_nnz": 3145728,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1241088,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4386816,
                            "linear_total": 12582912,
                            "nnz": 4395870,
                            "total": 12595456
                        },
                        "13": {
                            "linear_attention_nnz": 3145728,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1179648,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4325376,
                            "linear_total": 12582912,
                            "nnz": 4334400,
                            "total": 12595456
                        },
                        "14": {
                            "linear_attention_nnz": 2883584,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 909312,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3792896,
                            "linear_total": 12582912,
                            "nnz": 3801596,
                            "total": 12595264
                        },
                        "15": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 681984,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3303424,
                            "linear_total": 12582912,
                            "nnz": 3311821,
                            "total": 12595072
                        },
                        "16": {
                            "linear_attention_nnz": 2883584,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 473088,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3356672,
                            "linear_total": 12582912,
                            "nnz": 3365159,
                            "total": 12595264
                        },
                        "17": {
                            "linear_attention_nnz": 2883584,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 368640,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3252224,
                            "linear_total": 12582912,
                            "nnz": 3260660,
                            "total": 12595264
                        },
                        "18": {
                            "linear_attention_nnz": 2883584,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 321536,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3205120,
                            "linear_total": 12582912,
                            "nnz": 3213533,
                            "total": 12595264
                        },
                        "19": {
                            "linear_attention_nnz": 2097152,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 270336,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2367488,
                            "linear_total": 12582912,
                            "nnz": 2375300,
                            "total": 12594688
                        },
                        "2": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 286720,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 811008,
                            "linear_total": 12582912,
                            "nnz": 817676,
                            "total": 12593536
                        },
                        "20": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 112640,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 899072,
                            "linear_total": 12582912,
                            "nnz": 905847,
                            "total": 12593728
                        },
                        "21": {
                            "linear_attention_nnz": 1310720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 77824,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1388544,
                            "linear_total": 12582912,
                            "nnz": 1395686,
                            "total": 12594112
                        },
                        "22": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 79872,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 866304,
                            "linear_total": 12582912,
                            "nnz": 873063,
                            "total": 12593728
                        },
                        "23": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 182272,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1230848,
                            "linear_total": 12582912,
                            "nnz": 1237849,
                            "total": 12593920
                        },
                        "3": {
                            "linear_attention_nnz": 1310720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 413696,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1724416,
                            "linear_total": 12582912,
                            "nnz": 1731722,
                            "total": 12594112
                        },
                        "4": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 466944,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 991232,
                            "linear_total": 12582912,
                            "nnz": 997988,
                            "total": 12593536
                        },
                        "5": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 552960,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1077248,
                            "linear_total": 12582912,
                            "nnz": 1084046,
                            "total": 12593536
                        },
                        "6": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 608256,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1394688,
                            "linear_total": 12582912,
                            "nnz": 1401705,
                            "total": 12593728
                        },
                        "7": {
                            "linear_attention_nnz": 1310720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 438272,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1748992,
                            "linear_total": 12582912,
                            "nnz": 1756310,
                            "total": 12594112
                        },
                        "8": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 661504,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1710080,
                            "linear_total": 12582912,
                            "nnz": 1717315,
                            "total": 12593920
                        },
                        "9": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 747520,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2320384,
                            "linear_total": 12582912,
                            "nnz": 2328045,
                            "total": 12594304
                        }
                    },
                    "linear_nnz": 53983232,
                    "linear_sparsity": 82.12415907118056,
                    "linear_total": 301989888,
                    "nnz": 85952121,
                    "pruned_heads": {
                        "0": [
                            2,
                            3,
                            4,
                            7,
                            8,
                            9,
                            10,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            7,
                            8,
                            10,
                            12,
                            13,
                            14
                        ],
                        "11": [
                            0,
                            2,
                            4,
                            5,
                            8,
                            10
                        ],
                        "12": [
                            10,
                            3,
                            13,
                            6
                        ],
                        "13": [
                            2,
                            10,
                            4,
                            12
                        ],
                        "14": [
                            2,
                            3,
                            4,
                            8,
                            11
                        ],
                        "15": [
                            0,
                            5,
                            6,
                            7,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            8,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            11,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            11,
                            13
                        ],
                        "19": [
                            0,
                            2,
                            3,
                            4,
                            9,
                            10,
                            11,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "21": [
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "23": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            13,
                            14
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            12,
                            13,
                            15
                        ]
                    },
                    "total": 334050946,
                    "total_sparsity": 74.26975674542769
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_10",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_10",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_10",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 90.10843526218638,
            "fill_rate": 0.4766890914351851,
            "speedup": 1.034699920808227
        },
        {
            "annotate": "24",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 1024,
                    "initializer_range": 0.02,
                    "intermediate_size": 4096,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 16,
                    "num_hidden_layers": 24,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            7,
                            8,
                            10,
                            12,
                            13
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            8,
                            10
                        ],
                        "12": [
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            13
                        ],
                        "13": [
                            10,
                            2,
                            3,
                            12
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            11
                        ],
                        "15": [
                            0,
                            5,
                            6,
                            7,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            8,
                            10,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            11,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            11,
                            12,
                            13
                        ],
                        "19": [
                            0,
                            2,
                            3,
                            4,
                            9,
                            10,
                            11,
                            13,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "21": [
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "23": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            12,
                            13,
                            15
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 84.399243140965,
                    "f1": 90.84270784891945
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_10_d0.25/checkpoint-22500",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test_large/large_regu_10_d0.25/checkpoint-210000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 41.6732879486084,
                    "eval_elapsed_time": 48.981834520120174
                },
                "speedup": 0.9261182619659336,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 835584,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2408448,
                            "linear_total": 12582912,
                            "nnz": 2416152,
                            "total": 12594304
                        },
                        "1": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1275904,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1800192,
                            "linear_total": 12582912,
                            "nnz": 1807343,
                            "total": 12593536
                        },
                        "10": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2410496,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 5031936,
                            "linear_total": 12582912,
                            "nnz": 5041177,
                            "total": 12595072
                        },
                        "11": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2510848,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4870144,
                            "linear_total": 12582912,
                            "nnz": 4879242,
                            "total": 12594880
                        },
                        "12": {
                            "linear_attention_nnz": 2097152,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2660352,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4757504,
                            "linear_total": 12582912,
                            "nnz": 4766483,
                            "total": 12594688
                        },
                        "13": {
                            "linear_attention_nnz": 3145728,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2605056,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 5750784,
                            "linear_total": 12582912,
                            "nnz": 5760504,
                            "total": 12595456
                        },
                        "14": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2299904,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4921344,
                            "linear_total": 12582912,
                            "nnz": 4930531,
                            "total": 12595072
                        },
                        "15": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1699840,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4321280,
                            "linear_total": 12582912,
                            "nnz": 4330174,
                            "total": 12595072
                        },
                        "16": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1402880,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4024320,
                            "linear_total": 12582912,
                            "nnz": 4033069,
                            "total": 12595072
                        },
                        "17": {
                            "linear_attention_nnz": 3145728,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1097728,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4243456,
                            "linear_total": 12582912,
                            "nnz": 4252440,
                            "total": 12595456
                        },
                        "18": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 901120,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3260416,
                            "linear_total": 12582912,
                            "nnz": 3268728,
                            "total": 12594880
                        },
                        "19": {
                            "linear_attention_nnz": 1835008,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 739328,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2574336,
                            "linear_total": 12582912,
                            "nnz": 2582185,
                            "total": 12594496
                        },
                        "2": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1359872,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1884160,
                            "linear_total": 12582912,
                            "nnz": 1891352,
                            "total": 12593536
                        },
                        "20": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 358400,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1406976,
                            "linear_total": 12582912,
                            "nnz": 1414063,
                            "total": 12593920
                        },
                        "21": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 194560,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1243136,
                            "linear_total": 12582912,
                            "nnz": 1250143,
                            "total": 12593920
                        },
                        "22": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 180224,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 704512,
                            "linear_total": 12582912,
                            "nnz": 711128,
                            "total": 12593536
                        },
                        "23": {
                            "linear_attention_nnz": 1310720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 323584,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1634304,
                            "linear_total": 12582912,
                            "nnz": 1641566,
                            "total": 12594112
                        },
                        "3": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1685504,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2734080,
                            "linear_total": 12582912,
                            "nnz": 2741815,
                            "total": 12593920
                        },
                        "4": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1767424,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2291712,
                            "linear_total": 12582912,
                            "nnz": 2299103,
                            "total": 12593536
                        },
                        "5": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1873920,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2660352,
                            "linear_total": 12582912,
                            "nnz": 2667987,
                            "total": 12593728
                        },
                        "6": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2054144,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2578432,
                            "linear_total": 12582912,
                            "nnz": 2585963,
                            "total": 12593536
                        },
                        "7": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1773568,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2822144,
                            "linear_total": 12582912,
                            "nnz": 2829922,
                            "total": 12593920
                        },
                        "8": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1968128,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2492416,
                            "linear_total": 12582912,
                            "nnz": 2499905,
                            "total": 12593536
                        },
                        "9": {
                            "linear_attention_nnz": 1310720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1986560,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3297280,
                            "linear_total": 12582912,
                            "nnz": 3305354,
                            "total": 12594112
                        }
                    },
                    "linear_nnz": 73713664,
                    "linear_sparsity": 75.59068467881944,
                    "linear_total": 301989888,
                    "nnz": 105691291,
                    "pruned_heads": {
                        "0": [
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            7,
                            8,
                            10,
                            12,
                            13
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            8,
                            10
                        ],
                        "12": [
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            13
                        ],
                        "13": [
                            10,
                            2,
                            3,
                            12
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            11
                        ],
                        "15": [
                            0,
                            5,
                            6,
                            7,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            8,
                            10,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            11,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            11,
                            12,
                            13
                        ],
                        "19": [
                            0,
                            2,
                            3,
                            4,
                            9,
                            10,
                            11,
                            13,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "21": [
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "23": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            12,
                            13,
                            15
                        ]
                    },
                    "total": 334048258,
                    "total_sparsity": 68.36047233630538
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_10_d0.25",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 5,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_10_d0.25",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_10_d0.25",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 90.84270784891945,
            "fill_rate": 0.6509150752314815,
            "speedup": 0.9261182619659336
        },
        {
            "annotate": "24",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 1024,
                    "initializer_range": 0.02,
                    "intermediate_size": 4096,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 16,
                    "num_hidden_layers": 24,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            7,
                            8,
                            10,
                            12,
                            13
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            8,
                            10
                        ],
                        "12": [
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            13
                        ],
                        "13": [
                            10,
                            2,
                            3,
                            12
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            11
                        ],
                        "15": [
                            0,
                            5,
                            6,
                            7,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            8,
                            10,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            11,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            11,
                            12,
                            13
                        ],
                        "19": [
                            0,
                            2,
                            3,
                            4,
                            9,
                            10,
                            11,
                            13,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "21": [
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "23": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            12,
                            13,
                            15
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 84.20056764427625,
                    "f1": 90.73941291394593
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_10_d0.25/checkpoint-25000",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test_large/large_regu_10_d0.25/checkpoint-210000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 41.50353849792481,
                    "eval_elapsed_time": 49.06402187002823
                },
                "speedup": 0.929906085171529,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 835584,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2408448,
                            "linear_total": 12582912,
                            "nnz": 2416152,
                            "total": 12594304
                        },
                        "1": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1275904,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1800192,
                            "linear_total": 12582912,
                            "nnz": 1807343,
                            "total": 12593536
                        },
                        "10": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2410496,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 5031936,
                            "linear_total": 12582912,
                            "nnz": 5041177,
                            "total": 12595072
                        },
                        "11": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2510848,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4870144,
                            "linear_total": 12582912,
                            "nnz": 4879242,
                            "total": 12594880
                        },
                        "12": {
                            "linear_attention_nnz": 2097152,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2660352,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4757504,
                            "linear_total": 12582912,
                            "nnz": 4766483,
                            "total": 12594688
                        },
                        "13": {
                            "linear_attention_nnz": 3145728,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2605056,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 5750784,
                            "linear_total": 12582912,
                            "nnz": 5760504,
                            "total": 12595456
                        },
                        "14": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2299904,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4921344,
                            "linear_total": 12582912,
                            "nnz": 4930531,
                            "total": 12595072
                        },
                        "15": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1699840,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4321280,
                            "linear_total": 12582912,
                            "nnz": 4330174,
                            "total": 12595072
                        },
                        "16": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1402880,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4024320,
                            "linear_total": 12582912,
                            "nnz": 4033069,
                            "total": 12595072
                        },
                        "17": {
                            "linear_attention_nnz": 3145728,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1097728,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4243456,
                            "linear_total": 12582912,
                            "nnz": 4252440,
                            "total": 12595456
                        },
                        "18": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 901120,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3260416,
                            "linear_total": 12582912,
                            "nnz": 3268728,
                            "total": 12594880
                        },
                        "19": {
                            "linear_attention_nnz": 1835008,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 739328,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2574336,
                            "linear_total": 12582912,
                            "nnz": 2582185,
                            "total": 12594496
                        },
                        "2": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1359872,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1884160,
                            "linear_total": 12582912,
                            "nnz": 1891352,
                            "total": 12593536
                        },
                        "20": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 358400,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1406976,
                            "linear_total": 12582912,
                            "nnz": 1414063,
                            "total": 12593920
                        },
                        "21": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 194560,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1243136,
                            "linear_total": 12582912,
                            "nnz": 1250143,
                            "total": 12593920
                        },
                        "22": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 180224,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 704512,
                            "linear_total": 12582912,
                            "nnz": 711128,
                            "total": 12593536
                        },
                        "23": {
                            "linear_attention_nnz": 1310720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 323584,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1634304,
                            "linear_total": 12582912,
                            "nnz": 1641566,
                            "total": 12594112
                        },
                        "3": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1685504,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2734080,
                            "linear_total": 12582912,
                            "nnz": 2741815,
                            "total": 12593920
                        },
                        "4": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1767424,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2291712,
                            "linear_total": 12582912,
                            "nnz": 2299103,
                            "total": 12593536
                        },
                        "5": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1873920,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2660352,
                            "linear_total": 12582912,
                            "nnz": 2667987,
                            "total": 12593728
                        },
                        "6": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2054144,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2578432,
                            "linear_total": 12582912,
                            "nnz": 2585963,
                            "total": 12593536
                        },
                        "7": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1773568,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2822144,
                            "linear_total": 12582912,
                            "nnz": 2829922,
                            "total": 12593920
                        },
                        "8": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1968128,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2492416,
                            "linear_total": 12582912,
                            "nnz": 2499905,
                            "total": 12593536
                        },
                        "9": {
                            "linear_attention_nnz": 1310720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1986560,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3297280,
                            "linear_total": 12582912,
                            "nnz": 3305354,
                            "total": 12594112
                        }
                    },
                    "linear_nnz": 73713664,
                    "linear_sparsity": 75.59068467881944,
                    "linear_total": 301989888,
                    "nnz": 105691291,
                    "pruned_heads": {
                        "0": [
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            7,
                            8,
                            10,
                            12,
                            13
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            8,
                            10
                        ],
                        "12": [
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            13
                        ],
                        "13": [
                            10,
                            2,
                            3,
                            12
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            11
                        ],
                        "15": [
                            0,
                            5,
                            6,
                            7,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            8,
                            10,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            11,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            11,
                            12,
                            13
                        ],
                        "19": [
                            0,
                            2,
                            3,
                            4,
                            9,
                            10,
                            11,
                            13,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "21": [
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "23": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            12,
                            13,
                            15
                        ]
                    },
                    "total": 334048258,
                    "total_sparsity": 68.36047233630538
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_10_d0.25",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 5,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_10_d0.25",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_10_d0.25",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 90.73941291394593,
            "fill_rate": 0.6509150752314815,
            "speedup": 0.929906085171529
        },
        {
            "annotate": "24",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 1024,
                    "initializer_range": 0.02,
                    "intermediate_size": 4096,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 16,
                    "num_hidden_layers": 24,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            7,
                            8,
                            10,
                            12,
                            13
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            8,
                            10
                        ],
                        "12": [
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            13
                        ],
                        "13": [
                            10,
                            2,
                            3,
                            12
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            11
                        ],
                        "15": [
                            0,
                            5,
                            6,
                            7,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            8,
                            10,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            11,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            11,
                            12,
                            13
                        ],
                        "19": [
                            0,
                            2,
                            3,
                            4,
                            9,
                            10,
                            11,
                            13,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "21": [
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "23": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            12,
                            13,
                            15
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 84.2100283822138,
                    "f1": 90.70141124860059
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_10_d0.25/checkpoint-27665",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test_large/large_regu_10_d0.25/checkpoint-210000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 41.6272840423584,
                    "eval_elapsed_time": 49.02150737866759
                },
                "speedup": 0.9271417507348992,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 835584,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2408448,
                            "linear_total": 12582912,
                            "nnz": 2416152,
                            "total": 12594304
                        },
                        "1": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1275904,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1800192,
                            "linear_total": 12582912,
                            "nnz": 1807343,
                            "total": 12593536
                        },
                        "10": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2410496,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 5031936,
                            "linear_total": 12582912,
                            "nnz": 5041177,
                            "total": 12595072
                        },
                        "11": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2510848,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4870144,
                            "linear_total": 12582912,
                            "nnz": 4879242,
                            "total": 12594880
                        },
                        "12": {
                            "linear_attention_nnz": 2097152,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2660352,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4757504,
                            "linear_total": 12582912,
                            "nnz": 4766483,
                            "total": 12594688
                        },
                        "13": {
                            "linear_attention_nnz": 3145728,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2605056,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 5750784,
                            "linear_total": 12582912,
                            "nnz": 5760504,
                            "total": 12595456
                        },
                        "14": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2299904,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4921344,
                            "linear_total": 12582912,
                            "nnz": 4930531,
                            "total": 12595072
                        },
                        "15": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1699840,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4321280,
                            "linear_total": 12582912,
                            "nnz": 4330174,
                            "total": 12595072
                        },
                        "16": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1402880,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4024320,
                            "linear_total": 12582912,
                            "nnz": 4033069,
                            "total": 12595072
                        },
                        "17": {
                            "linear_attention_nnz": 3145728,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1097728,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4243456,
                            "linear_total": 12582912,
                            "nnz": 4252440,
                            "total": 12595456
                        },
                        "18": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 901120,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3260416,
                            "linear_total": 12582912,
                            "nnz": 3268728,
                            "total": 12594880
                        },
                        "19": {
                            "linear_attention_nnz": 1835008,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 739328,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2574336,
                            "linear_total": 12582912,
                            "nnz": 2582185,
                            "total": 12594496
                        },
                        "2": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1359872,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1884160,
                            "linear_total": 12582912,
                            "nnz": 1891352,
                            "total": 12593536
                        },
                        "20": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 358400,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1406976,
                            "linear_total": 12582912,
                            "nnz": 1414063,
                            "total": 12593920
                        },
                        "21": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 194560,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1243136,
                            "linear_total": 12582912,
                            "nnz": 1250143,
                            "total": 12593920
                        },
                        "22": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 180224,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 704512,
                            "linear_total": 12582912,
                            "nnz": 711128,
                            "total": 12593536
                        },
                        "23": {
                            "linear_attention_nnz": 1310720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 323584,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1634304,
                            "linear_total": 12582912,
                            "nnz": 1641566,
                            "total": 12594112
                        },
                        "3": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1685504,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2734080,
                            "linear_total": 12582912,
                            "nnz": 2741815,
                            "total": 12593920
                        },
                        "4": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1767424,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2291712,
                            "linear_total": 12582912,
                            "nnz": 2299103,
                            "total": 12593536
                        },
                        "5": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1873920,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2660352,
                            "linear_total": 12582912,
                            "nnz": 2667987,
                            "total": 12593728
                        },
                        "6": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2054144,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2578432,
                            "linear_total": 12582912,
                            "nnz": 2585963,
                            "total": 12593536
                        },
                        "7": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1773568,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2822144,
                            "linear_total": 12582912,
                            "nnz": 2829922,
                            "total": 12593920
                        },
                        "8": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1968128,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2492416,
                            "linear_total": 12582912,
                            "nnz": 2499905,
                            "total": 12593536
                        },
                        "9": {
                            "linear_attention_nnz": 1310720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1986560,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3297280,
                            "linear_total": 12582912,
                            "nnz": 3305354,
                            "total": 12594112
                        }
                    },
                    "linear_nnz": 73713664,
                    "linear_sparsity": 75.59068467881944,
                    "linear_total": 301989888,
                    "nnz": 105691291,
                    "pruned_heads": {
                        "0": [
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            7,
                            8,
                            10,
                            12,
                            13
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            8,
                            10
                        ],
                        "12": [
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            13
                        ],
                        "13": [
                            10,
                            2,
                            3,
                            12
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            11
                        ],
                        "15": [
                            0,
                            5,
                            6,
                            7,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            8,
                            10,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            11,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            11,
                            12,
                            13
                        ],
                        "19": [
                            0,
                            2,
                            3,
                            4,
                            9,
                            10,
                            11,
                            13,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "21": [
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "23": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            12,
                            13,
                            15
                        ]
                    },
                    "total": 334048258,
                    "total_sparsity": 68.36047233630538
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_10_d0.25",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 5,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_10_d0.25",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_10_d0.25",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 90.70141124860059,
            "fill_rate": 0.6509150752314815,
            "speedup": 0.9271417507348992
        },
        {
            "annotate": "24",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 1024,
                    "initializer_range": 0.02,
                    "intermediate_size": 4096,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 16,
                    "num_hidden_layers": 24,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            7,
                            8,
                            10,
                            12,
                            13
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            8,
                            10
                        ],
                        "12": [
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            13
                        ],
                        "13": [
                            10,
                            2,
                            3,
                            12
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            11
                        ],
                        "15": [
                            0,
                            5,
                            6,
                            7,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            8,
                            10,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            11,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            11,
                            12,
                            13
                        ],
                        "19": [
                            0,
                            2,
                            3,
                            4,
                            9,
                            10,
                            11,
                            13,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "21": [
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "23": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            12,
                            13,
                            15
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 84.63576158940397,
                    "f1": 91.0266636723574
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_10_d0.25_v3_f91.03/checkpoint-55000",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test_large/large_regu_10_d0.25/checkpoint-210000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 41.85157574462891,
                    "eval_elapsed_time": 49.32021534908563
                },
                "speedup": 0.9221729963255725,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 835584,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2408448,
                            "linear_total": 12582912,
                            "nnz": 2416152,
                            "total": 12594304
                        },
                        "1": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1275904,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1800192,
                            "linear_total": 12582912,
                            "nnz": 1807343,
                            "total": 12593536
                        },
                        "10": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2410496,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 5031936,
                            "linear_total": 12582912,
                            "nnz": 5041177,
                            "total": 12595072
                        },
                        "11": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2510848,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4870144,
                            "linear_total": 12582912,
                            "nnz": 4879242,
                            "total": 12594880
                        },
                        "12": {
                            "linear_attention_nnz": 2097152,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2660352,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4757504,
                            "linear_total": 12582912,
                            "nnz": 4766483,
                            "total": 12594688
                        },
                        "13": {
                            "linear_attention_nnz": 3145728,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2605056,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 5750784,
                            "linear_total": 12582912,
                            "nnz": 5760504,
                            "total": 12595456
                        },
                        "14": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2299904,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4921344,
                            "linear_total": 12582912,
                            "nnz": 4930531,
                            "total": 12595072
                        },
                        "15": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1699840,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4321280,
                            "linear_total": 12582912,
                            "nnz": 4330174,
                            "total": 12595072
                        },
                        "16": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1402880,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4024320,
                            "linear_total": 12582912,
                            "nnz": 4033069,
                            "total": 12595072
                        },
                        "17": {
                            "linear_attention_nnz": 3145728,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1097728,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4243456,
                            "linear_total": 12582912,
                            "nnz": 4252440,
                            "total": 12595456
                        },
                        "18": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 901120,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3260416,
                            "linear_total": 12582912,
                            "nnz": 3268728,
                            "total": 12594880
                        },
                        "19": {
                            "linear_attention_nnz": 1835008,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 739328,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2574336,
                            "linear_total": 12582912,
                            "nnz": 2582185,
                            "total": 12594496
                        },
                        "2": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1359872,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1884160,
                            "linear_total": 12582912,
                            "nnz": 1891352,
                            "total": 12593536
                        },
                        "20": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 358400,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1406976,
                            "linear_total": 12582912,
                            "nnz": 1414063,
                            "total": 12593920
                        },
                        "21": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 194560,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1243136,
                            "linear_total": 12582912,
                            "nnz": 1250143,
                            "total": 12593920
                        },
                        "22": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 180224,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 704512,
                            "linear_total": 12582912,
                            "nnz": 711128,
                            "total": 12593536
                        },
                        "23": {
                            "linear_attention_nnz": 1310720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 323584,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1634304,
                            "linear_total": 12582912,
                            "nnz": 1641566,
                            "total": 12594112
                        },
                        "3": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1685504,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2734080,
                            "linear_total": 12582912,
                            "nnz": 2741815,
                            "total": 12593920
                        },
                        "4": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1767424,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2291712,
                            "linear_total": 12582912,
                            "nnz": 2299103,
                            "total": 12593536
                        },
                        "5": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1873920,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2660352,
                            "linear_total": 12582912,
                            "nnz": 2667987,
                            "total": 12593728
                        },
                        "6": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2054144,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2578432,
                            "linear_total": 12582912,
                            "nnz": 2585963,
                            "total": 12593536
                        },
                        "7": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1773568,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2822144,
                            "linear_total": 12582912,
                            "nnz": 2829922,
                            "total": 12593920
                        },
                        "8": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1968128,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2492416,
                            "linear_total": 12582912,
                            "nnz": 2499905,
                            "total": 12593536
                        },
                        "9": {
                            "linear_attention_nnz": 1310720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1986560,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3297280,
                            "linear_total": 12582912,
                            "nnz": 3305354,
                            "total": 12594112
                        }
                    },
                    "linear_nnz": 73713664,
                    "linear_sparsity": 75.59068467881944,
                    "linear_total": 301989888,
                    "nnz": 105691291,
                    "pruned_heads": {
                        "0": [
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            7,
                            8,
                            10,
                            12,
                            13
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            8,
                            10
                        ],
                        "12": [
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            13
                        ],
                        "13": [
                            10,
                            2,
                            3,
                            12
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            11
                        ],
                        "15": [
                            0,
                            5,
                            6,
                            7,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            8,
                            10,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            11,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            11,
                            12,
                            13
                        ],
                        "19": [
                            0,
                            2,
                            3,
                            4,
                            9,
                            10,
                            11,
                            13,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "21": [
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "23": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            12,
                            13,
                            15
                        ]
                    },
                    "total": 334048258,
                    "total_sparsity": 68.36047233630538
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_10_d0.25",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_10_d0.25",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_10_d0.25",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 91.0266636723574,
            "fill_rate": 0.6509150752314815,
            "speedup": 0.9221729963255725
        },
        {
            "annotate": "24",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 1024,
                    "initializer_range": 0.02,
                    "intermediate_size": 4096,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 16,
                    "num_hidden_layers": 24,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            7,
                            8,
                            10,
                            12,
                            13
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            8,
                            10
                        ],
                        "12": [
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            13
                        ],
                        "13": [
                            10,
                            2,
                            3,
                            12
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            11
                        ],
                        "15": [
                            0,
                            5,
                            6,
                            7,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            8,
                            10,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            11,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            11,
                            12,
                            13
                        ],
                        "19": [
                            0,
                            2,
                            3,
                            4,
                            9,
                            10,
                            11,
                            13,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "21": [
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "23": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            12,
                            13,
                            15
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 84.65468306527909,
                    "f1": 91.01004624462917
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_10_d0.25_v3_f91.03/checkpoint-55330",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test_large/large_regu_10_d0.25/checkpoint-210000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 41.85431317138672,
                    "eval_elapsed_time": 49.428419118281454
                },
                "speedup": 0.922112682803639,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 835584,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2408448,
                            "linear_total": 12582912,
                            "nnz": 2416152,
                            "total": 12594304
                        },
                        "1": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1275904,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1800192,
                            "linear_total": 12582912,
                            "nnz": 1807343,
                            "total": 12593536
                        },
                        "10": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2410496,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 5031936,
                            "linear_total": 12582912,
                            "nnz": 5041177,
                            "total": 12595072
                        },
                        "11": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2510848,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4870144,
                            "linear_total": 12582912,
                            "nnz": 4879242,
                            "total": 12594880
                        },
                        "12": {
                            "linear_attention_nnz": 2097152,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2660352,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4757504,
                            "linear_total": 12582912,
                            "nnz": 4766483,
                            "total": 12594688
                        },
                        "13": {
                            "linear_attention_nnz": 3145728,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2605056,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 5750784,
                            "linear_total": 12582912,
                            "nnz": 5760504,
                            "total": 12595456
                        },
                        "14": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2299904,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4921344,
                            "linear_total": 12582912,
                            "nnz": 4930531,
                            "total": 12595072
                        },
                        "15": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1699840,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4321280,
                            "linear_total": 12582912,
                            "nnz": 4330174,
                            "total": 12595072
                        },
                        "16": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1402880,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4024320,
                            "linear_total": 12582912,
                            "nnz": 4033069,
                            "total": 12595072
                        },
                        "17": {
                            "linear_attention_nnz": 3145728,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1097728,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4243456,
                            "linear_total": 12582912,
                            "nnz": 4252440,
                            "total": 12595456
                        },
                        "18": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 901120,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3260416,
                            "linear_total": 12582912,
                            "nnz": 3268728,
                            "total": 12594880
                        },
                        "19": {
                            "linear_attention_nnz": 1835008,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 739328,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2574336,
                            "linear_total": 12582912,
                            "nnz": 2582185,
                            "total": 12594496
                        },
                        "2": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1359872,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1884160,
                            "linear_total": 12582912,
                            "nnz": 1891352,
                            "total": 12593536
                        },
                        "20": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 358400,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1406976,
                            "linear_total": 12582912,
                            "nnz": 1414063,
                            "total": 12593920
                        },
                        "21": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 194560,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1243136,
                            "linear_total": 12582912,
                            "nnz": 1250143,
                            "total": 12593920
                        },
                        "22": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 180224,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 704512,
                            "linear_total": 12582912,
                            "nnz": 711128,
                            "total": 12593536
                        },
                        "23": {
                            "linear_attention_nnz": 1310720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 323584,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1634304,
                            "linear_total": 12582912,
                            "nnz": 1641566,
                            "total": 12594112
                        },
                        "3": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1685504,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2734080,
                            "linear_total": 12582912,
                            "nnz": 2741815,
                            "total": 12593920
                        },
                        "4": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1767424,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2291712,
                            "linear_total": 12582912,
                            "nnz": 2299103,
                            "total": 12593536
                        },
                        "5": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1873920,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2660352,
                            "linear_total": 12582912,
                            "nnz": 2667987,
                            "total": 12593728
                        },
                        "6": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 2054144,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2578432,
                            "linear_total": 12582912,
                            "nnz": 2585963,
                            "total": 12593536
                        },
                        "7": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1773568,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2822144,
                            "linear_total": 12582912,
                            "nnz": 2829922,
                            "total": 12593920
                        },
                        "8": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1968128,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2492416,
                            "linear_total": 12582912,
                            "nnz": 2499905,
                            "total": 12593536
                        },
                        "9": {
                            "linear_attention_nnz": 1310720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1986560,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3297280,
                            "linear_total": 12582912,
                            "nnz": 3305354,
                            "total": 12594112
                        }
                    },
                    "linear_nnz": 73713664,
                    "linear_sparsity": 75.59068467881944,
                    "linear_total": 301989888,
                    "nnz": 105691291,
                    "pruned_heads": {
                        "0": [
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            7,
                            8,
                            10,
                            12,
                            13
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            8,
                            10
                        ],
                        "12": [
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            13
                        ],
                        "13": [
                            10,
                            2,
                            3,
                            12
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            11
                        ],
                        "15": [
                            0,
                            5,
                            6,
                            7,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            8,
                            10,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            11,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            11,
                            12,
                            13
                        ],
                        "19": [
                            0,
                            2,
                            3,
                            4,
                            9,
                            10,
                            11,
                            13,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "21": [
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "23": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            12,
                            13,
                            15
                        ]
                    },
                    "total": 334048258,
                    "total_sparsity": 68.36047233630538
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_10_d0.25",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_10_d0.25",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_10_d0.25",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 91.01004624462917,
            "fill_rate": 0.6509150752314815,
            "speedup": 0.922112682803639
        },
        {
            "annotate": "11",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 1024,
                    "initializer_range": 0.02,
                    "intermediate_size": 4096,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 16,
                    "num_hidden_layers": 24,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            14
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            12
                        ],
                        "12": [
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            14
                        ],
                        "13": [
                            2,
                            3,
                            4,
                            10,
                            11,
                            12
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            11
                        ],
                        "15": [
                            0,
                            2,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            8,
                            10,
                            12,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            11,
                            12,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            11,
                            12,
                            13
                        ],
                        "19": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "21": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "23": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            12,
                            13,
                            15
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 81.15421002838221,
                    "f1": 88.34901265417608
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_40/checkpoint-55330",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test_large/large_regu_40/checkpoint-221320",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 28.669108856201174,
                    "eval_elapsed_time": 35.70603838330135
                },
                "speedup": 1.3462013485997515,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 88064,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1136640,
                            "linear_total": 12582912,
                            "nnz": 1143595,
                            "total": 12593920
                        },
                        "1": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 102400,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 626688,
                            "linear_total": 12582912,
                            "nnz": 633266,
                            "total": 12593536
                        },
                        "10": {
                            "linear_attention_nnz": 1835008,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 442368,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2277376,
                            "linear_total": 12582912,
                            "nnz": 2285080,
                            "total": 12594496
                        },
                        "11": {
                            "linear_attention_nnz": 1835008,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 462848,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2297856,
                            "linear_total": 12582912,
                            "nnz": 2305570,
                            "total": 12594496
                        },
                        "12": {
                            "linear_attention_nnz": 1310720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 557056,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1867776,
                            "linear_total": 12582912,
                            "nnz": 1875152,
                            "total": 12594112
                        },
                        "13": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 507904,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3129344,
                            "linear_total": 12582912,
                            "nnz": 3137656,
                            "total": 12595072
                        },
                        "14": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 362496,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2983936,
                            "linear_total": 12582912,
                            "nnz": 2992177,
                            "total": 12595072
                        },
                        "15": {
                            "linear_attention_nnz": 1835008,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 278528,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2113536,
                            "linear_total": 12582912,
                            "nnz": 2121160,
                            "total": 12594496
                        },
                        "16": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 188416,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2547712,
                            "linear_total": 12582912,
                            "nnz": 2555676,
                            "total": 12594880
                        },
                        "17": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 188416,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2809856,
                            "linear_total": 12582912,
                            "nnz": 2818012,
                            "total": 12595072
                        },
                        "18": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 141312,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2500608,
                            "linear_total": 12582912,
                            "nnz": 2508549,
                            "total": 12594880
                        },
                        "19": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 137216,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1185792,
                            "linear_total": 12582912,
                            "nnz": 1192771,
                            "total": 12593920
                        },
                        "2": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 90112,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 352256,
                            "linear_total": 12582912,
                            "nnz": 358636,
                            "total": 12593344
                        },
                        "20": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 57344,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 843776,
                            "linear_total": 12582912,
                            "nnz": 850524,
                            "total": 12593728
                        },
                        "21": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 40960,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 827392,
                            "linear_total": 12582912,
                            "nnz": 834132,
                            "total": 12593728
                        },
                        "22": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 40960,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 565248,
                            "linear_total": 12582912,
                            "nnz": 571796,
                            "total": 12593536
                        },
                        "23": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 102400,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 888832,
                            "linear_total": 12582912,
                            "nnz": 895602,
                            "total": 12593728
                        },
                        "3": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 155648,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 679936,
                            "linear_total": 12582912,
                            "nnz": 686540,
                            "total": 12593536
                        },
                        "4": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 143360,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 667648,
                            "linear_total": 12582912,
                            "nnz": 674246,
                            "total": 12593536
                        },
                        "5": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 167936,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 692224,
                            "linear_total": 12582912,
                            "nnz": 698834,
                            "total": 12593536
                        },
                        "6": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 212992,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 475136,
                            "linear_total": 12582912,
                            "nnz": 481576,
                            "total": 12593344
                        },
                        "7": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 178176,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1226752,
                            "linear_total": 12582912,
                            "nnz": 1233751,
                            "total": 12593920
                        },
                        "8": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 229376,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 753664,
                            "linear_total": 12582912,
                            "nnz": 760304,
                            "total": 12593536
                        },
                        "9": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 370688,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1419264,
                            "linear_total": 12582912,
                            "nnz": 1426357,
                            "total": 12593920
                        }
                    },
                    "linear_nnz": 34869248,
                    "linear_sparsity": 88.45350477430556,
                    "linear_total": 301989888,
                    "nnz": 66825924,
                    "pruned_heads": {
                        "0": [
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            14
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            12
                        ],
                        "12": [
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            14
                        ],
                        "13": [
                            2,
                            3,
                            4,
                            10,
                            11,
                            12
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            11
                        ],
                        "15": [
                            0,
                            2,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            8,
                            10,
                            12,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            11,
                            12,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            11,
                            12,
                            13
                        ],
                        "19": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "21": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "23": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            12,
                            13,
                            15
                        ]
                    },
                    "total": 334042306,
                    "total_sparsity": 79.99477227893404
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_40",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_40",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_40",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.34901265417608,
            "fill_rate": 0.30790653935185175,
            "speedup": 1.3462013485997515
        },
        {
            "annotate": "14",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 1024,
                    "initializer_range": 0.02,
                    "intermediate_size": 4096,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 16,
                    "num_hidden_layers": 24,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            14
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            12,
                            15
                        ],
                        "12": [
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "13": [
                            2,
                            3,
                            4,
                            10,
                            11,
                            12
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            11,
                            13
                        ],
                        "15": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            11,
                            12,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            11,
                            12,
                            13
                        ],
                        "19": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            9,
                            10,
                            11,
                            13,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "21": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "23": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12,
                            13,
                            15
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 82.32734153263955,
                    "f1": 89.39825688878855
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_40_d0.25/checkpoint-52500",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test_large/large_regu_40_d0.25/checkpoint-220000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 29.977725273132325,
                    "eval_elapsed_time": 37.05464425915852
                },
                "speedup": 1.2874356761138743,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 253952,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 778240,
                            "linear_total": 12582912,
                            "nnz": 784892,
                            "total": 12593536
                        },
                        "1": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 432128,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 956416,
                            "linear_total": 12582912,
                            "nnz": 963155,
                            "total": 12593536
                        },
                        "10": {
                            "linear_attention_nnz": 1835008,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1210368,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3045376,
                            "linear_total": 12582912,
                            "nnz": 3053455,
                            "total": 12594496
                        },
                        "11": {
                            "linear_attention_nnz": 1310720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1277952,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2588672,
                            "linear_total": 12582912,
                            "nnz": 2596400,
                            "total": 12594112
                        },
                        "12": {
                            "linear_attention_nnz": 1310720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1400832,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2711552,
                            "linear_total": 12582912,
                            "nnz": 2719340,
                            "total": 12594112
                        },
                        "13": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1464320,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4085760,
                            "linear_total": 12582912,
                            "nnz": 4094539,
                            "total": 12595072
                        },
                        "14": {
                            "linear_attention_nnz": 2097152,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1122304,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3219456,
                            "linear_total": 12582912,
                            "nnz": 3227684,
                            "total": 12594688
                        },
                        "15": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 778240,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2351104,
                            "linear_total": 12582912,
                            "nnz": 2358780,
                            "total": 12594304
                        },
                        "16": {
                            "linear_attention_nnz": 2097152,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 532480,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2629632,
                            "linear_total": 12582912,
                            "nnz": 2637572,
                            "total": 12594688
                        },
                        "17": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 456704,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3078144,
                            "linear_total": 12582912,
                            "nnz": 3086431,
                            "total": 12595072
                        },
                        "18": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 440320,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2799616,
                            "linear_total": 12582912,
                            "nnz": 2807703,
                            "total": 12594880
                        },
                        "19": {
                            "linear_attention_nnz": 1310720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 362496,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1673216,
                            "linear_total": 12582912,
                            "nnz": 1680497,
                            "total": 12594112
                        },
                        "2": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 450560,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 974848,
                            "linear_total": 12582912,
                            "nnz": 981596,
                            "total": 12593536
                        },
                        "20": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 184320,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 708608,
                            "linear_total": 12582912,
                            "nnz": 715226,
                            "total": 12593536
                        },
                        "21": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 112640,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 899072,
                            "linear_total": 12582912,
                            "nnz": 905847,
                            "total": 12593728
                        },
                        "22": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 114688,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 901120,
                            "linear_total": 12582912,
                            "nnz": 907896,
                            "total": 12593728
                        },
                        "23": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 184320,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 708608,
                            "linear_total": 12582912,
                            "nnz": 715226,
                            "total": 12593536
                        },
                        "3": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 548864,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1073152,
                            "linear_total": 12582912,
                            "nnz": 1079948,
                            "total": 12593536
                        },
                        "4": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 614400,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1138688,
                            "linear_total": 12582912,
                            "nnz": 1145516,
                            "total": 12593536
                        },
                        "5": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 839680,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1101824,
                            "linear_total": 12582912,
                            "nnz": 1108570,
                            "total": 12593344
                        },
                        "6": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 858112,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1120256,
                            "linear_total": 12582912,
                            "nnz": 1127011,
                            "total": 12593344
                        },
                        "7": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 636928,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1423360,
                            "linear_total": 12582912,
                            "nnz": 1430391,
                            "total": 12593728
                        },
                        "8": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 847872,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1372160,
                            "linear_total": 12582912,
                            "nnz": 1379102,
                            "total": 12593536
                        },
                        "9": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 901120,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1687552,
                            "linear_total": 12582912,
                            "nnz": 1694712,
                            "total": 12593728
                        }
                    },
                    "linear_nnz": 43026432,
                    "linear_sparsity": 85.75236002604166,
                    "linear_total": 301989888,
                    "nnz": 74986451,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            14
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            12,
                            15
                        ],
                        "12": [
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "13": [
                            2,
                            3,
                            4,
                            10,
                            11,
                            12
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            11,
                            13
                        ],
                        "15": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            11,
                            12,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            11,
                            12,
                            13
                        ],
                        "19": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            9,
                            10,
                            11,
                            13,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "21": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "23": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12,
                            13,
                            15
                        ]
                    },
                    "total": 334040386,
                    "total_sparsity": 77.5516811311552
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_40_d0.25",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_40_d0.25",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_40_d0.25",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 89.39825688878855,
            "fill_rate": 0.3799370659722226,
            "speedup": 1.2874356761138743
        },
        {
            "annotate": "14",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 1024,
                    "initializer_range": 0.02,
                    "intermediate_size": 4096,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 16,
                    "num_hidden_layers": 24,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            14
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            12,
                            15
                        ],
                        "12": [
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "13": [
                            2,
                            3,
                            4,
                            10,
                            11,
                            12
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            11,
                            13
                        ],
                        "15": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            11,
                            12,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            11,
                            12,
                            13
                        ],
                        "19": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            9,
                            10,
                            11,
                            13,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "21": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "23": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12,
                            13,
                            15
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 82.28003784295176,
                    "f1": 89.37602873453882
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_40_d0.25/checkpoint-55000",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test_large/large_regu_40_d0.25/checkpoint-220000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 29.986146453857423,
                    "eval_elapsed_time": 37.03868922078982
                },
                "speedup": 1.287074118201884,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 253952,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 778240,
                            "linear_total": 12582912,
                            "nnz": 784892,
                            "total": 12593536
                        },
                        "1": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 432128,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 956416,
                            "linear_total": 12582912,
                            "nnz": 963155,
                            "total": 12593536
                        },
                        "10": {
                            "linear_attention_nnz": 1835008,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1210368,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3045376,
                            "linear_total": 12582912,
                            "nnz": 3053455,
                            "total": 12594496
                        },
                        "11": {
                            "linear_attention_nnz": 1310720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1277952,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2588672,
                            "linear_total": 12582912,
                            "nnz": 2596400,
                            "total": 12594112
                        },
                        "12": {
                            "linear_attention_nnz": 1310720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1400832,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2711552,
                            "linear_total": 12582912,
                            "nnz": 2719340,
                            "total": 12594112
                        },
                        "13": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1464320,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4085760,
                            "linear_total": 12582912,
                            "nnz": 4094539,
                            "total": 12595072
                        },
                        "14": {
                            "linear_attention_nnz": 2097152,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1122304,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3219456,
                            "linear_total": 12582912,
                            "nnz": 3227684,
                            "total": 12594688
                        },
                        "15": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 778240,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2351104,
                            "linear_total": 12582912,
                            "nnz": 2358780,
                            "total": 12594304
                        },
                        "16": {
                            "linear_attention_nnz": 2097152,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 532480,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2629632,
                            "linear_total": 12582912,
                            "nnz": 2637572,
                            "total": 12594688
                        },
                        "17": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 456704,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3078144,
                            "linear_total": 12582912,
                            "nnz": 3086431,
                            "total": 12595072
                        },
                        "18": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 440320,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2799616,
                            "linear_total": 12582912,
                            "nnz": 2807703,
                            "total": 12594880
                        },
                        "19": {
                            "linear_attention_nnz": 1310720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 362496,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1673216,
                            "linear_total": 12582912,
                            "nnz": 1680497,
                            "total": 12594112
                        },
                        "2": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 450560,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 974848,
                            "linear_total": 12582912,
                            "nnz": 981596,
                            "total": 12593536
                        },
                        "20": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 184320,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 708608,
                            "linear_total": 12582912,
                            "nnz": 715226,
                            "total": 12593536
                        },
                        "21": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 112640,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 899072,
                            "linear_total": 12582912,
                            "nnz": 905847,
                            "total": 12593728
                        },
                        "22": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 114688,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 901120,
                            "linear_total": 12582912,
                            "nnz": 907896,
                            "total": 12593728
                        },
                        "23": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 184320,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 708608,
                            "linear_total": 12582912,
                            "nnz": 715226,
                            "total": 12593536
                        },
                        "3": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 548864,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1073152,
                            "linear_total": 12582912,
                            "nnz": 1079948,
                            "total": 12593536
                        },
                        "4": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 614400,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1138688,
                            "linear_total": 12582912,
                            "nnz": 1145516,
                            "total": 12593536
                        },
                        "5": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 839680,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1101824,
                            "linear_total": 12582912,
                            "nnz": 1108570,
                            "total": 12593344
                        },
                        "6": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 858112,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1120256,
                            "linear_total": 12582912,
                            "nnz": 1127011,
                            "total": 12593344
                        },
                        "7": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 636928,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1423360,
                            "linear_total": 12582912,
                            "nnz": 1430391,
                            "total": 12593728
                        },
                        "8": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 847872,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1372160,
                            "linear_total": 12582912,
                            "nnz": 1379102,
                            "total": 12593536
                        },
                        "9": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 901120,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1687552,
                            "linear_total": 12582912,
                            "nnz": 1694712,
                            "total": 12593728
                        }
                    },
                    "linear_nnz": 43026432,
                    "linear_sparsity": 85.75236002604166,
                    "linear_total": 301989888,
                    "nnz": 74986451,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            14
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            12,
                            15
                        ],
                        "12": [
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "13": [
                            2,
                            3,
                            4,
                            10,
                            11,
                            12
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            11,
                            13
                        ],
                        "15": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            11,
                            12,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            11,
                            12,
                            13
                        ],
                        "19": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            9,
                            10,
                            11,
                            13,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "21": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "23": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12,
                            13,
                            15
                        ]
                    },
                    "total": 334040386,
                    "total_sparsity": 77.5516811311552
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_40_d0.25",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_40_d0.25",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_40_d0.25",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 89.37602873453882,
            "fill_rate": 0.3799370659722226,
            "speedup": 1.287074118201884
        },
        {
            "annotate": "14",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 1024,
                    "initializer_range": 0.02,
                    "intermediate_size": 4096,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 16,
                    "num_hidden_layers": 24,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            14
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            12,
                            15
                        ],
                        "12": [
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "13": [
                            2,
                            3,
                            4,
                            10,
                            11,
                            12
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            11,
                            13
                        ],
                        "15": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            11,
                            12,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            11,
                            12,
                            13
                        ],
                        "19": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            9,
                            10,
                            11,
                            13,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "21": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "23": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12,
                            13,
                            15
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 82.25165562913908,
                    "f1": 89.36914341970648
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_40_d0.25/checkpoint-55330",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test_large/large_regu_40_d0.25/checkpoint-220000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 29.99512833404541,
                    "eval_elapsed_time": 37.09979658899829
                },
                "speedup": 1.2866887107652496,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 253952,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 778240,
                            "linear_total": 12582912,
                            "nnz": 784892,
                            "total": 12593536
                        },
                        "1": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 432128,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 956416,
                            "linear_total": 12582912,
                            "nnz": 963155,
                            "total": 12593536
                        },
                        "10": {
                            "linear_attention_nnz": 1835008,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1210368,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3045376,
                            "linear_total": 12582912,
                            "nnz": 3053455,
                            "total": 12594496
                        },
                        "11": {
                            "linear_attention_nnz": 1310720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1277952,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2588672,
                            "linear_total": 12582912,
                            "nnz": 2596400,
                            "total": 12594112
                        },
                        "12": {
                            "linear_attention_nnz": 1310720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1400832,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2711552,
                            "linear_total": 12582912,
                            "nnz": 2719340,
                            "total": 12594112
                        },
                        "13": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1464320,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 4085760,
                            "linear_total": 12582912,
                            "nnz": 4094539,
                            "total": 12595072
                        },
                        "14": {
                            "linear_attention_nnz": 2097152,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1122304,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3219456,
                            "linear_total": 12582912,
                            "nnz": 3227684,
                            "total": 12594688
                        },
                        "15": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 778240,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2351104,
                            "linear_total": 12582912,
                            "nnz": 2358780,
                            "total": 12594304
                        },
                        "16": {
                            "linear_attention_nnz": 2097152,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 532480,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2629632,
                            "linear_total": 12582912,
                            "nnz": 2637572,
                            "total": 12594688
                        },
                        "17": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 456704,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3078144,
                            "linear_total": 12582912,
                            "nnz": 3086431,
                            "total": 12595072
                        },
                        "18": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 440320,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2799616,
                            "linear_total": 12582912,
                            "nnz": 2807703,
                            "total": 12594880
                        },
                        "19": {
                            "linear_attention_nnz": 1310720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 362496,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1673216,
                            "linear_total": 12582912,
                            "nnz": 1680497,
                            "total": 12594112
                        },
                        "2": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 450560,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 974848,
                            "linear_total": 12582912,
                            "nnz": 981596,
                            "total": 12593536
                        },
                        "20": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 184320,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 708608,
                            "linear_total": 12582912,
                            "nnz": 715226,
                            "total": 12593536
                        },
                        "21": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 112640,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 899072,
                            "linear_total": 12582912,
                            "nnz": 905847,
                            "total": 12593728
                        },
                        "22": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 114688,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 901120,
                            "linear_total": 12582912,
                            "nnz": 907896,
                            "total": 12593728
                        },
                        "23": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 184320,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 708608,
                            "linear_total": 12582912,
                            "nnz": 715226,
                            "total": 12593536
                        },
                        "3": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 548864,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1073152,
                            "linear_total": 12582912,
                            "nnz": 1079948,
                            "total": 12593536
                        },
                        "4": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 614400,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1138688,
                            "linear_total": 12582912,
                            "nnz": 1145516,
                            "total": 12593536
                        },
                        "5": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 839680,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1101824,
                            "linear_total": 12582912,
                            "nnz": 1108570,
                            "total": 12593344
                        },
                        "6": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 858112,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1120256,
                            "linear_total": 12582912,
                            "nnz": 1127011,
                            "total": 12593344
                        },
                        "7": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 636928,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1423360,
                            "linear_total": 12582912,
                            "nnz": 1430391,
                            "total": 12593728
                        },
                        "8": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 847872,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1372160,
                            "linear_total": 12582912,
                            "nnz": 1379102,
                            "total": 12593536
                        },
                        "9": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 901120,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1687552,
                            "linear_total": 12582912,
                            "nnz": 1694712,
                            "total": 12593728
                        }
                    },
                    "linear_nnz": 43026432,
                    "linear_sparsity": 85.75236002604166,
                    "linear_total": 301989888,
                    "nnz": 74986451,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            14
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            12,
                            15
                        ],
                        "12": [
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "13": [
                            2,
                            3,
                            4,
                            10,
                            11,
                            12
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            11,
                            13
                        ],
                        "15": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            11,
                            12,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            11,
                            12,
                            13
                        ],
                        "19": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            9,
                            10,
                            11,
                            13,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "21": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "23": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12,
                            13,
                            15
                        ]
                    },
                    "total": 334040386,
                    "total_sparsity": 77.5516811311552
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_40_d0.25",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_40_d0.25",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_40_d0.25",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 89.36914341970648,
            "fill_rate": 0.3799370659722226,
            "speedup": 1.2866887107652496
        },
        {
            "annotate": "9",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 1024,
                    "initializer_range": 0.02,
                    "intermediate_size": 4096,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 16,
                    "num_hidden_layers": 24,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            3,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            12,
                            15
                        ],
                        "12": [
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "13": [
                            2,
                            3,
                            4,
                            9,
                            10,
                            11,
                            12
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            11,
                            12
                        ],
                        "15": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            8,
                            10,
                            12,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            8,
                            11,
                            12,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            11,
                            13
                        ],
                        "19": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "2": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "21": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "23": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12,
                            13,
                            15
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.81078524124882,
                    "f1": 87.48044078095904
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_60/checkpoint-45000",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test_large/large_regu_60/checkpoint-221320",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 26.35342041015625,
                    "eval_elapsed_time": 33.25582229997963
                },
                "speedup": 1.4644927453324936,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 71680,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 858112,
                            "linear_total": 12582912,
                            "nnz": 864867,
                            "total": 12593728
                        },
                        "1": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 59392,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 321536,
                            "linear_total": 12582912,
                            "nnz": 327901,
                            "total": 12593344
                        },
                        "10": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 360448,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1933312,
                            "linear_total": 12582912,
                            "nnz": 1940784,
                            "total": 12594304
                        },
                        "11": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 352256,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1925120,
                            "linear_total": 12582912,
                            "nnz": 1932588,
                            "total": 12594304
                        },
                        "12": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 460800,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1509376,
                            "linear_total": 12582912,
                            "nnz": 1516513,
                            "total": 12593920
                        },
                        "13": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 382976,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2742272,
                            "linear_total": 12582912,
                            "nnz": 2750331,
                            "total": 12594880
                        },
                        "14": {
                            "linear_attention_nnz": 2097152,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 276480,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2373632,
                            "linear_total": 12582912,
                            "nnz": 2381447,
                            "total": 12594688
                        },
                        "15": {
                            "linear_attention_nnz": 1310720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 258048,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1568768,
                            "linear_total": 12582912,
                            "nnz": 1575998,
                            "total": 12594112
                        },
                        "16": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 147456,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2506752,
                            "linear_total": 12582912,
                            "nnz": 2514696,
                            "total": 12594880
                        },
                        "17": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 145408,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2504704,
                            "linear_total": 12582912,
                            "nnz": 2512647,
                            "total": 12594880
                        },
                        "18": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 116736,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2738176,
                            "linear_total": 12582912,
                            "nnz": 2746297,
                            "total": 12595072
                        },
                        "19": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 100352,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 886784,
                            "linear_total": 12582912,
                            "nnz": 893553,
                            "total": 12593728
                        },
                        "2": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 63488,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 325632,
                            "linear_total": 12582912,
                            "nnz": 331999,
                            "total": 12593344
                        },
                        "20": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 49152,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 835584,
                            "linear_total": 12582912,
                            "nnz": 842328,
                            "total": 12593728
                        },
                        "21": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 40960,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 827392,
                            "linear_total": 12582912,
                            "nnz": 834132,
                            "total": 12593728
                        },
                        "22": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 49152,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 573440,
                            "linear_total": 12582912,
                            "nnz": 579992,
                            "total": 12593536
                        },
                        "23": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 81920,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 606208,
                            "linear_total": 12582912,
                            "nnz": 612776,
                            "total": 12593536
                        },
                        "3": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 100352,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 624640,
                            "linear_total": 12582912,
                            "nnz": 631217,
                            "total": 12593536
                        },
                        "4": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 133120,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 657408,
                            "linear_total": 12582912,
                            "nnz": 664001,
                            "total": 12593536
                        },
                        "5": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 129024,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 391168,
                            "linear_total": 12582912,
                            "nnz": 397567,
                            "total": 12593344
                        },
                        "6": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 155648,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 417792,
                            "linear_total": 12582912,
                            "nnz": 424204,
                            "total": 12593344
                        },
                        "7": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 145408,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1193984,
                            "linear_total": 12582912,
                            "nnz": 1200967,
                            "total": 12593920
                        },
                        "8": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 174080,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 436224,
                            "linear_total": 12582912,
                            "nnz": 442645,
                            "total": 12593344
                        },
                        "9": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 274432,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1060864,
                            "linear_total": 12582912,
                            "nnz": 1067718,
                            "total": 12593728
                        }
                    },
                    "linear_nnz": 29818880,
                    "linear_sparsity": 90.12586805555556,
                    "linear_total": 301989888,
                    "nnz": 61772130,
                    "pruned_heads": {
                        "0": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            3,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            12,
                            15
                        ],
                        "12": [
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "13": [
                            2,
                            3,
                            4,
                            9,
                            10,
                            11,
                            12
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            11,
                            12
                        ],
                        "15": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            8,
                            10,
                            12,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            8,
                            11,
                            12,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            11,
                            13
                        ],
                        "19": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "2": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "21": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "23": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12,
                            13,
                            15
                        ]
                    },
                    "total": 334039426,
                    "total_sparsity": 81.50753318561863
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_60",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_60",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_60",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.48044078095904,
            "fill_rate": 0.2633101851851851,
            "speedup": 1.4644927453324936
        },
        {
            "annotate": "9",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 1024,
                    "initializer_range": 0.02,
                    "intermediate_size": 4096,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 16,
                    "num_hidden_layers": 24,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            3,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            12,
                            15
                        ],
                        "12": [
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "13": [
                            2,
                            3,
                            4,
                            9,
                            10,
                            11,
                            12
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            11,
                            12
                        ],
                        "15": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            8,
                            10,
                            12,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            8,
                            11,
                            12,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            11,
                            13
                        ],
                        "19": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "2": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "21": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "23": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12,
                            13,
                            15
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.84862819299906,
                    "f1": 87.41551421723396
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_60/checkpoint-55000",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test_large/large_regu_60/checkpoint-221320",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 26.059397384643557,
                    "eval_elapsed_time": 33.108247430063784
                },
                "speedup": 1.4810163272660417,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 71680,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 858112,
                            "linear_total": 12582912,
                            "nnz": 864867,
                            "total": 12593728
                        },
                        "1": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 59392,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 321536,
                            "linear_total": 12582912,
                            "nnz": 327901,
                            "total": 12593344
                        },
                        "10": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 360448,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1933312,
                            "linear_total": 12582912,
                            "nnz": 1940784,
                            "total": 12594304
                        },
                        "11": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 352256,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1925120,
                            "linear_total": 12582912,
                            "nnz": 1932588,
                            "total": 12594304
                        },
                        "12": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 460800,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1509376,
                            "linear_total": 12582912,
                            "nnz": 1516513,
                            "total": 12593920
                        },
                        "13": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 382976,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2742272,
                            "linear_total": 12582912,
                            "nnz": 2750331,
                            "total": 12594880
                        },
                        "14": {
                            "linear_attention_nnz": 2097152,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 276480,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2373632,
                            "linear_total": 12582912,
                            "nnz": 2381447,
                            "total": 12594688
                        },
                        "15": {
                            "linear_attention_nnz": 1310720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 258048,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1568768,
                            "linear_total": 12582912,
                            "nnz": 1575998,
                            "total": 12594112
                        },
                        "16": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 147456,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2506752,
                            "linear_total": 12582912,
                            "nnz": 2514696,
                            "total": 12594880
                        },
                        "17": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 145408,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2504704,
                            "linear_total": 12582912,
                            "nnz": 2512647,
                            "total": 12594880
                        },
                        "18": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 116736,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2738176,
                            "linear_total": 12582912,
                            "nnz": 2746297,
                            "total": 12595072
                        },
                        "19": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 100352,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 886784,
                            "linear_total": 12582912,
                            "nnz": 893553,
                            "total": 12593728
                        },
                        "2": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 63488,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 325632,
                            "linear_total": 12582912,
                            "nnz": 331999,
                            "total": 12593344
                        },
                        "20": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 49152,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 835584,
                            "linear_total": 12582912,
                            "nnz": 842328,
                            "total": 12593728
                        },
                        "21": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 40960,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 827392,
                            "linear_total": 12582912,
                            "nnz": 834132,
                            "total": 12593728
                        },
                        "22": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 49152,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 573440,
                            "linear_total": 12582912,
                            "nnz": 579992,
                            "total": 12593536
                        },
                        "23": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 81920,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 606208,
                            "linear_total": 12582912,
                            "nnz": 612776,
                            "total": 12593536
                        },
                        "3": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 100352,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 624640,
                            "linear_total": 12582912,
                            "nnz": 631217,
                            "total": 12593536
                        },
                        "4": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 133120,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 657408,
                            "linear_total": 12582912,
                            "nnz": 664001,
                            "total": 12593536
                        },
                        "5": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 129024,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 391168,
                            "linear_total": 12582912,
                            "nnz": 397567,
                            "total": 12593344
                        },
                        "6": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 155648,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 417792,
                            "linear_total": 12582912,
                            "nnz": 424204,
                            "total": 12593344
                        },
                        "7": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 145408,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1193984,
                            "linear_total": 12582912,
                            "nnz": 1200967,
                            "total": 12593920
                        },
                        "8": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 174080,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 436224,
                            "linear_total": 12582912,
                            "nnz": 442645,
                            "total": 12593344
                        },
                        "9": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 274432,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1060864,
                            "linear_total": 12582912,
                            "nnz": 1067718,
                            "total": 12593728
                        }
                    },
                    "linear_nnz": 29818880,
                    "linear_sparsity": 90.12586805555556,
                    "linear_total": 301989888,
                    "nnz": 61772130,
                    "pruned_heads": {
                        "0": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            3,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            12,
                            15
                        ],
                        "12": [
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "13": [
                            2,
                            3,
                            4,
                            9,
                            10,
                            11,
                            12
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            11,
                            12
                        ],
                        "15": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            8,
                            10,
                            12,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            8,
                            11,
                            12,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            11,
                            13
                        ],
                        "19": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "2": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "21": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "23": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12,
                            13,
                            15
                        ]
                    },
                    "total": 334039426,
                    "total_sparsity": 81.50753318561863
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_60",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_60",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_60",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.41551421723396,
            "fill_rate": 0.2633101851851851,
            "speedup": 1.4810163272660417
        },
        {
            "annotate": "9",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 1024,
                    "initializer_range": 0.02,
                    "intermediate_size": 4096,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 16,
                    "num_hidden_layers": 24,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            3,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            12,
                            15
                        ],
                        "12": [
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "13": [
                            2,
                            3,
                            4,
                            9,
                            10,
                            11,
                            12
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            11,
                            12
                        ],
                        "15": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            8,
                            10,
                            12,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            8,
                            11,
                            12,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            11,
                            13
                        ],
                        "19": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "2": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "21": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "23": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12,
                            13,
                            15
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.7918637653737,
                    "f1": 87.39314667097808
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_60/checkpoint-55330",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test_large/large_regu_60/checkpoint-221320",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 26.185509422302246,
                    "eval_elapsed_time": 33.24473914410919
                },
                "speedup": 1.4738836042082184,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 71680,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 858112,
                            "linear_total": 12582912,
                            "nnz": 864867,
                            "total": 12593728
                        },
                        "1": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 59392,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 321536,
                            "linear_total": 12582912,
                            "nnz": 327901,
                            "total": 12593344
                        },
                        "10": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 360448,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1933312,
                            "linear_total": 12582912,
                            "nnz": 1940784,
                            "total": 12594304
                        },
                        "11": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 352256,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1925120,
                            "linear_total": 12582912,
                            "nnz": 1932588,
                            "total": 12594304
                        },
                        "12": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 460800,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1509376,
                            "linear_total": 12582912,
                            "nnz": 1516513,
                            "total": 12593920
                        },
                        "13": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 382976,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2742272,
                            "linear_total": 12582912,
                            "nnz": 2750331,
                            "total": 12594880
                        },
                        "14": {
                            "linear_attention_nnz": 2097152,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 276480,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2373632,
                            "linear_total": 12582912,
                            "nnz": 2381447,
                            "total": 12594688
                        },
                        "15": {
                            "linear_attention_nnz": 1310720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 258048,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1568768,
                            "linear_total": 12582912,
                            "nnz": 1575998,
                            "total": 12594112
                        },
                        "16": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 147456,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2506752,
                            "linear_total": 12582912,
                            "nnz": 2514696,
                            "total": 12594880
                        },
                        "17": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 145408,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2504704,
                            "linear_total": 12582912,
                            "nnz": 2512647,
                            "total": 12594880
                        },
                        "18": {
                            "linear_attention_nnz": 2621440,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 116736,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2738176,
                            "linear_total": 12582912,
                            "nnz": 2746297,
                            "total": 12595072
                        },
                        "19": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 100352,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 886784,
                            "linear_total": 12582912,
                            "nnz": 893553,
                            "total": 12593728
                        },
                        "2": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 63488,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 325632,
                            "linear_total": 12582912,
                            "nnz": 331999,
                            "total": 12593344
                        },
                        "20": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 49152,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 835584,
                            "linear_total": 12582912,
                            "nnz": 842328,
                            "total": 12593728
                        },
                        "21": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 40960,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 827392,
                            "linear_total": 12582912,
                            "nnz": 834132,
                            "total": 12593728
                        },
                        "22": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 49152,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 573440,
                            "linear_total": 12582912,
                            "nnz": 579992,
                            "total": 12593536
                        },
                        "23": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 81920,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 606208,
                            "linear_total": 12582912,
                            "nnz": 612776,
                            "total": 12593536
                        },
                        "3": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 100352,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 624640,
                            "linear_total": 12582912,
                            "nnz": 631217,
                            "total": 12593536
                        },
                        "4": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 133120,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 657408,
                            "linear_total": 12582912,
                            "nnz": 664001,
                            "total": 12593536
                        },
                        "5": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 129024,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 391168,
                            "linear_total": 12582912,
                            "nnz": 397567,
                            "total": 12593344
                        },
                        "6": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 155648,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 417792,
                            "linear_total": 12582912,
                            "nnz": 424204,
                            "total": 12593344
                        },
                        "7": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 145408,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1193984,
                            "linear_total": 12582912,
                            "nnz": 1200967,
                            "total": 12593920
                        },
                        "8": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 174080,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 436224,
                            "linear_total": 12582912,
                            "nnz": 442645,
                            "total": 12593344
                        },
                        "9": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 274432,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1060864,
                            "linear_total": 12582912,
                            "nnz": 1067718,
                            "total": 12593728
                        }
                    },
                    "linear_nnz": 29818880,
                    "linear_sparsity": 90.12586805555556,
                    "linear_total": 301989888,
                    "nnz": 61772130,
                    "pruned_heads": {
                        "0": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            3,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            12,
                            15
                        ],
                        "12": [
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "13": [
                            2,
                            3,
                            4,
                            9,
                            10,
                            11,
                            12
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            11,
                            12
                        ],
                        "15": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            8,
                            10,
                            12,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            8,
                            11,
                            12,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            11,
                            13
                        ],
                        "19": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "2": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "21": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "23": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12,
                            13,
                            15
                        ]
                    },
                    "total": 334039426,
                    "total_sparsity": 81.50753318561863
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_60",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_60",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_60",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.39314667097808,
            "fill_rate": 0.2633101851851851,
            "speedup": 1.4738836042082184
        },
        {
            "annotate": "12",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 1024,
                    "initializer_range": 0.02,
                    "intermediate_size": 4096,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 16,
                    "num_hidden_layers": 24,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            12,
                            15
                        ],
                        "12": [
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "13": [
                            2,
                            3,
                            4,
                            9,
                            10,
                            11,
                            12,
                            13
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            11,
                            12
                        ],
                        "15": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            8,
                            11,
                            12,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "19": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "21": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "23": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12,
                            13,
                            15
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 81.31504257332072,
                    "f1": 88.43872986679673
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_60_d0.25/checkpoint-47500",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test_large/large_regu_60_d0.25/checkpoint-221320",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 27.657082702636718,
                    "eval_elapsed_time": 34.62446285132319
                },
                "speedup": 1.3954614599207766,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 231424,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1017856,
                            "linear_total": 12582912,
                            "nnz": 1024689,
                            "total": 12593728
                        },
                        "1": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 290816,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 552960,
                            "linear_total": 12582912,
                            "nnz": 559438,
                            "total": 12593344
                        },
                        "10": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 923648,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2496512,
                            "linear_total": 12582912,
                            "nnz": 2504259,
                            "total": 12594304
                        },
                        "11": {
                            "linear_attention_nnz": 1310720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1075200,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2385920,
                            "linear_total": 12582912,
                            "nnz": 2393549,
                            "total": 12594112
                        },
                        "12": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1126400,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2174976,
                            "linear_total": 12582912,
                            "nnz": 2182438,
                            "total": 12593920
                        },
                        "13": {
                            "linear_attention_nnz": 2097152,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1179648,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3276800,
                            "linear_total": 12582912,
                            "nnz": 3285056,
                            "total": 12594688
                        },
                        "14": {
                            "linear_attention_nnz": 2097152,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 929792,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3026944,
                            "linear_total": 12582912,
                            "nnz": 3035078,
                            "total": 12594688
                        },
                        "15": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 622592,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2195456,
                            "linear_total": 12582912,
                            "nnz": 2203056,
                            "total": 12594304
                        },
                        "16": {
                            "linear_attention_nnz": 2097152,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 403456,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2500608,
                            "linear_total": 12582912,
                            "nnz": 2508485,
                            "total": 12594688
                        },
                        "17": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 360448,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2719744,
                            "linear_total": 12582912,
                            "nnz": 2727792,
                            "total": 12594880
                        },
                        "18": {
                            "linear_attention_nnz": 1835008,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 335872,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2170880,
                            "linear_total": 12582912,
                            "nnz": 2178532,
                            "total": 12594496
                        },
                        "19": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 307200,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1093632,
                            "linear_total": 12582912,
                            "nnz": 1100502,
                            "total": 12593728
                        },
                        "2": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 331776,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 856064,
                            "linear_total": 12582912,
                            "nnz": 862754,
                            "total": 12593536
                        },
                        "20": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 165888,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 690176,
                            "linear_total": 12582912,
                            "nnz": 696785,
                            "total": 12593536
                        },
                        "21": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 75776,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 862208,
                            "linear_total": 12582912,
                            "nnz": 868965,
                            "total": 12593728
                        },
                        "22": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 100352,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 624640,
                            "linear_total": 12582912,
                            "nnz": 631217,
                            "total": 12593536
                        },
                        "23": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 184320,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 446464,
                            "linear_total": 12582912,
                            "nnz": 452890,
                            "total": 12593344
                        },
                        "3": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 432128,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 956416,
                            "linear_total": 12582912,
                            "nnz": 963155,
                            "total": 12593536
                        },
                        "4": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 452608,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 976896,
                            "linear_total": 12582912,
                            "nnz": 983645,
                            "total": 12593536
                        },
                        "5": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 614400,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 876544,
                            "linear_total": 12582912,
                            "nnz": 883180,
                            "total": 12593344
                        },
                        "6": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 598016,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 860160,
                            "linear_total": 12582912,
                            "nnz": 866788,
                            "total": 12593344
                        },
                        "7": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 466944,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1253376,
                            "linear_total": 12582912,
                            "nnz": 1260324,
                            "total": 12593728
                        },
                        "8": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 673792,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 935936,
                            "linear_total": 12582912,
                            "nnz": 942601,
                            "total": 12593344
                        },
                        "9": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 692224,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1478656,
                            "linear_total": 12582912,
                            "nnz": 1485714,
                            "total": 12593728
                        }
                    },
                    "linear_nnz": 36429824,
                    "linear_sparsity": 87.93674045138889,
                    "linear_total": 301989888,
                    "nnz": 68385854,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            12,
                            15
                        ],
                        "12": [
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "13": [
                            2,
                            3,
                            4,
                            9,
                            10,
                            11,
                            12,
                            13
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            11,
                            12
                        ],
                        "15": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            8,
                            11,
                            12,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "19": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "21": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "23": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12,
                            13,
                            15
                        ]
                    },
                    "total": 334038082,
                    "total_sparsity": 79.52752764279134
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_60_d0.25",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_60_d0.25",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_60_d0.25",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.43872986679673,
            "fill_rate": 0.3216869212962964,
            "speedup": 1.3954614599207766
        },
        {
            "annotate": "12",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 1024,
                    "initializer_range": 0.02,
                    "intermediate_size": 4096,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 16,
                    "num_hidden_layers": 24,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            12,
                            15
                        ],
                        "12": [
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "13": [
                            2,
                            3,
                            4,
                            9,
                            10,
                            11,
                            12,
                            13
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            11,
                            12
                        ],
                        "15": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            8,
                            11,
                            12,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "19": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "21": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "23": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12,
                            13,
                            15
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 81.49479659413434,
                    "f1": 88.43605833215561
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_60_d0.25/checkpoint-50000",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test_large/large_regu_60_d0.25/checkpoint-221320",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 27.367526008605957,
                    "eval_elapsed_time": 34.362684624269605
                },
                "speedup": 1.4102258637932692,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 231424,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1017856,
                            "linear_total": 12582912,
                            "nnz": 1024689,
                            "total": 12593728
                        },
                        "1": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 290816,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 552960,
                            "linear_total": 12582912,
                            "nnz": 559438,
                            "total": 12593344
                        },
                        "10": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 923648,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2496512,
                            "linear_total": 12582912,
                            "nnz": 2504259,
                            "total": 12594304
                        },
                        "11": {
                            "linear_attention_nnz": 1310720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1075200,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2385920,
                            "linear_total": 12582912,
                            "nnz": 2393549,
                            "total": 12594112
                        },
                        "12": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1126400,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2174976,
                            "linear_total": 12582912,
                            "nnz": 2182438,
                            "total": 12593920
                        },
                        "13": {
                            "linear_attention_nnz": 2097152,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1179648,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3276800,
                            "linear_total": 12582912,
                            "nnz": 3285056,
                            "total": 12594688
                        },
                        "14": {
                            "linear_attention_nnz": 2097152,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 929792,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3026944,
                            "linear_total": 12582912,
                            "nnz": 3035078,
                            "total": 12594688
                        },
                        "15": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 622592,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2195456,
                            "linear_total": 12582912,
                            "nnz": 2203056,
                            "total": 12594304
                        },
                        "16": {
                            "linear_attention_nnz": 2097152,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 403456,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2500608,
                            "linear_total": 12582912,
                            "nnz": 2508485,
                            "total": 12594688
                        },
                        "17": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 360448,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2719744,
                            "linear_total": 12582912,
                            "nnz": 2727792,
                            "total": 12594880
                        },
                        "18": {
                            "linear_attention_nnz": 1835008,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 335872,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2170880,
                            "linear_total": 12582912,
                            "nnz": 2178532,
                            "total": 12594496
                        },
                        "19": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 307200,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1093632,
                            "linear_total": 12582912,
                            "nnz": 1100502,
                            "total": 12593728
                        },
                        "2": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 331776,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 856064,
                            "linear_total": 12582912,
                            "nnz": 862754,
                            "total": 12593536
                        },
                        "20": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 165888,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 690176,
                            "linear_total": 12582912,
                            "nnz": 696785,
                            "total": 12593536
                        },
                        "21": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 75776,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 862208,
                            "linear_total": 12582912,
                            "nnz": 868965,
                            "total": 12593728
                        },
                        "22": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 100352,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 624640,
                            "linear_total": 12582912,
                            "nnz": 631217,
                            "total": 12593536
                        },
                        "23": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 184320,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 446464,
                            "linear_total": 12582912,
                            "nnz": 452890,
                            "total": 12593344
                        },
                        "3": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 432128,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 956416,
                            "linear_total": 12582912,
                            "nnz": 963155,
                            "total": 12593536
                        },
                        "4": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 452608,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 976896,
                            "linear_total": 12582912,
                            "nnz": 983645,
                            "total": 12593536
                        },
                        "5": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 614400,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 876544,
                            "linear_total": 12582912,
                            "nnz": 883180,
                            "total": 12593344
                        },
                        "6": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 598016,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 860160,
                            "linear_total": 12582912,
                            "nnz": 866788,
                            "total": 12593344
                        },
                        "7": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 466944,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1253376,
                            "linear_total": 12582912,
                            "nnz": 1260324,
                            "total": 12593728
                        },
                        "8": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 673792,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 935936,
                            "linear_total": 12582912,
                            "nnz": 942601,
                            "total": 12593344
                        },
                        "9": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 692224,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1478656,
                            "linear_total": 12582912,
                            "nnz": 1485714,
                            "total": 12593728
                        }
                    },
                    "linear_nnz": 36429824,
                    "linear_sparsity": 87.93674045138889,
                    "linear_total": 301989888,
                    "nnz": 68385854,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            12,
                            15
                        ],
                        "12": [
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "13": [
                            2,
                            3,
                            4,
                            9,
                            10,
                            11,
                            12,
                            13
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            11,
                            12
                        ],
                        "15": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            8,
                            11,
                            12,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "19": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "21": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "23": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12,
                            13,
                            15
                        ]
                    },
                    "total": 334038082,
                    "total_sparsity": 79.52752764279134
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_60_d0.25",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_60_d0.25",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_60_d0.25",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.43605833215561,
            "fill_rate": 0.3216869212962964,
            "speedup": 1.4102258637932692
        },
        {
            "annotate": "12",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 1024,
                    "initializer_range": 0.02,
                    "intermediate_size": 4096,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 16,
                    "num_hidden_layers": 24,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            12,
                            15
                        ],
                        "12": [
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "13": [
                            2,
                            3,
                            4,
                            9,
                            10,
                            11,
                            12,
                            13
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            11,
                            12
                        ],
                        "15": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            8,
                            11,
                            12,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "19": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "21": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "23": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12,
                            13,
                            15
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 81.41911069063387,
                    "f1": 88.38806203813442
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_60_d0.25/checkpoint-55000",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test_large/large_regu_60_d0.25/checkpoint-221320",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 27.473532768249513,
                    "eval_elapsed_time": 34.37180141918361
                },
                "speedup": 1.4047845004474155,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 231424,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1017856,
                            "linear_total": 12582912,
                            "nnz": 1024689,
                            "total": 12593728
                        },
                        "1": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 290816,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 552960,
                            "linear_total": 12582912,
                            "nnz": 559438,
                            "total": 12593344
                        },
                        "10": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 923648,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2496512,
                            "linear_total": 12582912,
                            "nnz": 2504259,
                            "total": 12594304
                        },
                        "11": {
                            "linear_attention_nnz": 1310720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1075200,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2385920,
                            "linear_total": 12582912,
                            "nnz": 2393549,
                            "total": 12594112
                        },
                        "12": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1126400,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2174976,
                            "linear_total": 12582912,
                            "nnz": 2182438,
                            "total": 12593920
                        },
                        "13": {
                            "linear_attention_nnz": 2097152,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1179648,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3276800,
                            "linear_total": 12582912,
                            "nnz": 3285056,
                            "total": 12594688
                        },
                        "14": {
                            "linear_attention_nnz": 2097152,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 929792,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3026944,
                            "linear_total": 12582912,
                            "nnz": 3035078,
                            "total": 12594688
                        },
                        "15": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 622592,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2195456,
                            "linear_total": 12582912,
                            "nnz": 2203056,
                            "total": 12594304
                        },
                        "16": {
                            "linear_attention_nnz": 2097152,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 403456,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2500608,
                            "linear_total": 12582912,
                            "nnz": 2508485,
                            "total": 12594688
                        },
                        "17": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 360448,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2719744,
                            "linear_total": 12582912,
                            "nnz": 2727792,
                            "total": 12594880
                        },
                        "18": {
                            "linear_attention_nnz": 1835008,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 335872,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2170880,
                            "linear_total": 12582912,
                            "nnz": 2178532,
                            "total": 12594496
                        },
                        "19": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 307200,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1093632,
                            "linear_total": 12582912,
                            "nnz": 1100502,
                            "total": 12593728
                        },
                        "2": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 331776,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 856064,
                            "linear_total": 12582912,
                            "nnz": 862754,
                            "total": 12593536
                        },
                        "20": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 165888,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 690176,
                            "linear_total": 12582912,
                            "nnz": 696785,
                            "total": 12593536
                        },
                        "21": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 75776,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 862208,
                            "linear_total": 12582912,
                            "nnz": 868965,
                            "total": 12593728
                        },
                        "22": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 100352,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 624640,
                            "linear_total": 12582912,
                            "nnz": 631217,
                            "total": 12593536
                        },
                        "23": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 184320,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 446464,
                            "linear_total": 12582912,
                            "nnz": 452890,
                            "total": 12593344
                        },
                        "3": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 432128,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 956416,
                            "linear_total": 12582912,
                            "nnz": 963155,
                            "total": 12593536
                        },
                        "4": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 452608,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 976896,
                            "linear_total": 12582912,
                            "nnz": 983645,
                            "total": 12593536
                        },
                        "5": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 614400,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 876544,
                            "linear_total": 12582912,
                            "nnz": 883180,
                            "total": 12593344
                        },
                        "6": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 598016,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 860160,
                            "linear_total": 12582912,
                            "nnz": 866788,
                            "total": 12593344
                        },
                        "7": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 466944,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1253376,
                            "linear_total": 12582912,
                            "nnz": 1260324,
                            "total": 12593728
                        },
                        "8": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 673792,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 935936,
                            "linear_total": 12582912,
                            "nnz": 942601,
                            "total": 12593344
                        },
                        "9": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 692224,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1478656,
                            "linear_total": 12582912,
                            "nnz": 1485714,
                            "total": 12593728
                        }
                    },
                    "linear_nnz": 36429824,
                    "linear_sparsity": 87.93674045138889,
                    "linear_total": 301989888,
                    "nnz": 68385854,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            12,
                            15
                        ],
                        "12": [
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "13": [
                            2,
                            3,
                            4,
                            9,
                            10,
                            11,
                            12,
                            13
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            11,
                            12
                        ],
                        "15": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            8,
                            11,
                            12,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "19": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "21": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "23": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12,
                            13,
                            15
                        ]
                    },
                    "total": 334038082,
                    "total_sparsity": 79.52752764279134
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_60_d0.25",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_60_d0.25",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_60_d0.25",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.38806203813442,
            "fill_rate": 0.3216869212962964,
            "speedup": 1.4047845004474155
        },
        {
            "annotate": "12",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 1024,
                    "initializer_range": 0.02,
                    "intermediate_size": 4096,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 16,
                    "num_hidden_layers": 24,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            12,
                            15
                        ],
                        "12": [
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "13": [
                            2,
                            3,
                            4,
                            9,
                            10,
                            11,
                            12,
                            13
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            11,
                            12
                        ],
                        "15": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            8,
                            11,
                            12,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "19": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "21": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "23": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12,
                            13,
                            15
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 81.41911069063387,
                    "f1": 88.38489115657516
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_60_d0.25/checkpoint-55330",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squad_test_large/large_regu_60_d0.25/checkpoint-221320",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 27.55610238647461,
                    "eval_elapsed_time": 34.603971847333014
                },
                "speedup": 1.4005751780162647,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 231424,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1017856,
                            "linear_total": 12582912,
                            "nnz": 1024689,
                            "total": 12593728
                        },
                        "1": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 290816,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 552960,
                            "linear_total": 12582912,
                            "nnz": 559438,
                            "total": 12593344
                        },
                        "10": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 923648,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2496512,
                            "linear_total": 12582912,
                            "nnz": 2504259,
                            "total": 12594304
                        },
                        "11": {
                            "linear_attention_nnz": 1310720,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1075200,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2385920,
                            "linear_total": 12582912,
                            "nnz": 2393549,
                            "total": 12594112
                        },
                        "12": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1126400,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2174976,
                            "linear_total": 12582912,
                            "nnz": 2182438,
                            "total": 12593920
                        },
                        "13": {
                            "linear_attention_nnz": 2097152,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 1179648,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3276800,
                            "linear_total": 12582912,
                            "nnz": 3285056,
                            "total": 12594688
                        },
                        "14": {
                            "linear_attention_nnz": 2097152,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 929792,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 3026944,
                            "linear_total": 12582912,
                            "nnz": 3035078,
                            "total": 12594688
                        },
                        "15": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 622592,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2195456,
                            "linear_total": 12582912,
                            "nnz": 2203056,
                            "total": 12594304
                        },
                        "16": {
                            "linear_attention_nnz": 2097152,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 403456,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2500608,
                            "linear_total": 12582912,
                            "nnz": 2508485,
                            "total": 12594688
                        },
                        "17": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 360448,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2719744,
                            "linear_total": 12582912,
                            "nnz": 2727792,
                            "total": 12594880
                        },
                        "18": {
                            "linear_attention_nnz": 1835008,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 335872,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 2170880,
                            "linear_total": 12582912,
                            "nnz": 2178532,
                            "total": 12594496
                        },
                        "19": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 307200,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1093632,
                            "linear_total": 12582912,
                            "nnz": 1100502,
                            "total": 12593728
                        },
                        "2": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 331776,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 856064,
                            "linear_total": 12582912,
                            "nnz": 862754,
                            "total": 12593536
                        },
                        "20": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 165888,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 690176,
                            "linear_total": 12582912,
                            "nnz": 696785,
                            "total": 12593536
                        },
                        "21": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 75776,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 862208,
                            "linear_total": 12582912,
                            "nnz": 868965,
                            "total": 12593728
                        },
                        "22": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 100352,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 624640,
                            "linear_total": 12582912,
                            "nnz": 631217,
                            "total": 12593536
                        },
                        "23": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 184320,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 446464,
                            "linear_total": 12582912,
                            "nnz": 452890,
                            "total": 12593344
                        },
                        "3": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 432128,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 956416,
                            "linear_total": 12582912,
                            "nnz": 963155,
                            "total": 12593536
                        },
                        "4": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 452608,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 976896,
                            "linear_total": 12582912,
                            "nnz": 983645,
                            "total": 12593536
                        },
                        "5": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 614400,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 876544,
                            "linear_total": 12582912,
                            "nnz": 883180,
                            "total": 12593344
                        },
                        "6": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 598016,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 860160,
                            "linear_total": 12582912,
                            "nnz": 866788,
                            "total": 12593344
                        },
                        "7": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 466944,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1253376,
                            "linear_total": 12582912,
                            "nnz": 1260324,
                            "total": 12593728
                        },
                        "8": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 673792,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 935936,
                            "linear_total": 12582912,
                            "nnz": 942601,
                            "total": 12593344
                        },
                        "9": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 4194304,
                            "linear_dense_nnz": 692224,
                            "linear_dense_total": 8388608,
                            "linear_nnz": 1478656,
                            "linear_total": 12582912,
                            "nnz": 1485714,
                            "total": 12593728
                        }
                    },
                    "linear_nnz": 36429824,
                    "linear_sparsity": 87.93674045138889,
                    "linear_total": 301989888,
                    "nnz": 68385854,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "10": [
                            0,
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            12,
                            15
                        ],
                        "12": [
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14
                        ],
                        "13": [
                            2,
                            3,
                            4,
                            9,
                            10,
                            11,
                            12,
                            13
                        ],
                        "14": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            11,
                            12
                        ],
                        "15": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12
                        ],
                        "16": [
                            3,
                            6,
                            7,
                            8,
                            10,
                            12,
                            13,
                            15
                        ],
                        "17": [
                            0,
                            2,
                            4,
                            8,
                            11,
                            12,
                            15
                        ],
                        "18": [
                            2,
                            3,
                            5,
                            9,
                            10,
                            11,
                            12,
                            13,
                            15
                        ],
                        "19": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            14,
                            15
                        ],
                        "20": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "21": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "22": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "23": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14
                        ],
                        "3": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            12,
                            13,
                            14,
                            15
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "5": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            13,
                            14,
                            15
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11,
                            12,
                            13,
                            15
                        ]
                    },
                    "total": 334038082,
                    "total_sparsity": 79.52752764279134
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_60_d0.25",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_60_d0.25",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_final_fine_tune/fine_tuned_large_regu_60_d0.25",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.38489115657516,
            "fill_rate": 0.3216869212962964,
            "speedup": 1.4005751780162647
        }
    ],
    "Block/unstruct method, bs= 16x16, dl=1.0": [
        {
            "annotate": "l=(10.0, 1.0, 1.0)",
            "cat_fun_name": "is_block_unstructured",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.93661305581836,
                    "f1": 88.29241912882233
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v8-a16-l10--2021-01-19--16-57-25/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 16,
                    "attention_block_rows": 16,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 29.73566310119629,
                    "eval_elapsed_time": 37.101448519621044
                },
                "speedup": 1.2979160032189903,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 469248,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2896466,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3365714,
                            "linear_total": 7077888,
                            "nnz": 3374066,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 574976,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2933134,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3508110,
                            "linear_total": 7077888,
                            "nnz": 3516606,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 355584,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1168698,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1524282,
                            "linear_total": 7077888,
                            "nnz": 1532777,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 201472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 632483,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 833955,
                            "linear_total": 7077888,
                            "nnz": 841723,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 634624,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3005666,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3640290,
                            "linear_total": 7077888,
                            "nnz": 3648946,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 851456,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2985914,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3837370,
                            "linear_total": 7077888,
                            "nnz": 3846282,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 830720,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2938982,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3769702,
                            "linear_total": 7077888,
                            "nnz": 3778630,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 756480,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2915873,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3672353,
                            "linear_total": 7077888,
                            "nnz": 3681089,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 768256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2787463,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3555719,
                            "linear_total": 7077888,
                            "nnz": 3564615,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 753408,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2497485,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3250893,
                            "linear_total": 7077888,
                            "nnz": 3259773,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 550912,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2115685,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2666597,
                            "linear_total": 7077888,
                            "nnz": 2675237,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 413184,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1490132,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1903316,
                            "linear_total": 7077888,
                            "nnz": 1911764,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 35528301,
                    "linear_sparsity": 58.16984176635742,
                    "linear_total": 84934656,
                    "nnz": 59470230,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            4,
                            7,
                            8
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            3,
                            4,
                            5,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 45.38663787466004
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.29241912882233,
            "fill_rate": 0.4183015823364258,
            "speedup": 1.2979160032189903
        },
        {
            "annotate": "l=(10.0, 1.0, 1.0)",
            "cat_fun_name": "is_block_unstructured",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.77578051087986,
                    "f1": 88.22778160568927
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v8-a16-l10--2021-01-19--16-57-25/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 16,
                    "attention_block_rows": 16,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 29.759838722229006,
                    "eval_elapsed_time": 37.11843426898122
                },
                "speedup": 1.2968616317313288,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 469248,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2896466,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3365714,
                            "linear_total": 7077888,
                            "nnz": 3374066,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 574976,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2933134,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3508110,
                            "linear_total": 7077888,
                            "nnz": 3516606,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 355584,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1168698,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1524282,
                            "linear_total": 7077888,
                            "nnz": 1532777,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 201472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 632483,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 833955,
                            "linear_total": 7077888,
                            "nnz": 841723,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 634624,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3005666,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3640290,
                            "linear_total": 7077888,
                            "nnz": 3648946,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 851456,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2985914,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3837370,
                            "linear_total": 7077888,
                            "nnz": 3846282,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 830720,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2938982,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3769702,
                            "linear_total": 7077888,
                            "nnz": 3778630,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 756480,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2915873,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3672353,
                            "linear_total": 7077888,
                            "nnz": 3681089,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 768256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2787463,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3555719,
                            "linear_total": 7077888,
                            "nnz": 3564615,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 753408,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2497485,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3250893,
                            "linear_total": 7077888,
                            "nnz": 3259773,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 550912,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2115685,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2666597,
                            "linear_total": 7077888,
                            "nnz": 2675237,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 413184,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1490132,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1903316,
                            "linear_total": 7077888,
                            "nnz": 1911764,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 35528301,
                    "linear_sparsity": 58.16984176635742,
                    "linear_total": 84934656,
                    "nnz": 59470230,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            4,
                            7,
                            8
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            3,
                            4,
                            5,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 45.38663787466004
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.22778160568927,
            "fill_rate": 0.4183015823364258,
            "speedup": 1.2968616317313288
        },
        {
            "annotate": "l=(20.0, 1.0, 1.0)",
            "cat_fun_name": "is_block_unstructured",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.92431409649953,
                    "f1": 87.57193515884181
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v8-a16-l20--2021-01-19--16-57-51/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 16,
                    "attention_block_rows": 16,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 27.83310959625244,
                    "eval_elapsed_time": 35.16166925104335
                },
                "speedup": 1.3866360448121684,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 335872,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2332233,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2668105,
                            "linear_total": 7077888,
                            "nnz": 2676329,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 451584,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2387496,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2839080,
                            "linear_total": 7077888,
                            "nnz": 2847464,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 224768,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 646159,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 870927,
                            "linear_total": 7077888,
                            "nnz": 879194,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 124672,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 325999,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 450671,
                            "linear_total": 7077888,
                            "nnz": 456932,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 408576,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2458332,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2866908,
                            "linear_total": 7077888,
                            "nnz": 2875228,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 682496,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2421186,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3103682,
                            "linear_total": 7077888,
                            "nnz": 3112498,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 504832,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2348406,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2853238,
                            "linear_total": 7077888,
                            "nnz": 2861766,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 558336,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2322448,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2880784,
                            "linear_total": 7077888,
                            "nnz": 2889408,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 475904,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2138474,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2614378,
                            "linear_total": 7077888,
                            "nnz": 2623018,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 542720,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1830088,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2372808,
                            "linear_total": 7077888,
                            "nnz": 2381496,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 312576,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1440402,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1752978,
                            "linear_total": 7077888,
                            "nnz": 1761282,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 288000,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 859129,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1147129,
                            "linear_total": 7077888,
                            "nnz": 1155416,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 26420688,
                    "linear_sparsity": 68.89292399088542,
                    "linear_total": 84934656,
                    "nnz": 50358753,
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 53.75399063078199
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.57193515884181,
            "fill_rate": 0.31107076009114576,
            "speedup": 1.3866360448121684
        },
        {
            "annotate": "l=(20.0, 1.0, 1.0)",
            "cat_fun_name": "is_block_unstructured",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.02838221381268,
                    "f1": 87.5280353923367
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v8-a16-l20--2021-01-19--16-57-51/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 16,
                    "attention_block_rows": 16,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 27.96729845428467,
                    "eval_elapsed_time": 35.3477450478822
                },
                "speedup": 1.3799828778048573,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 335872,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2332233,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2668105,
                            "linear_total": 7077888,
                            "nnz": 2676329,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 451584,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2387496,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2839080,
                            "linear_total": 7077888,
                            "nnz": 2847464,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 224768,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 646159,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 870927,
                            "linear_total": 7077888,
                            "nnz": 879194,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 124672,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 325999,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 450671,
                            "linear_total": 7077888,
                            "nnz": 456932,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 408576,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2458332,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2866908,
                            "linear_total": 7077888,
                            "nnz": 2875228,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 682496,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2421186,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3103682,
                            "linear_total": 7077888,
                            "nnz": 3112498,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 504832,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2348406,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2853238,
                            "linear_total": 7077888,
                            "nnz": 2861766,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 558336,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2322448,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2880784,
                            "linear_total": 7077888,
                            "nnz": 2889408,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 475904,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2138474,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2614378,
                            "linear_total": 7077888,
                            "nnz": 2623018,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 542720,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1830088,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2372808,
                            "linear_total": 7077888,
                            "nnz": 2381496,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 312576,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1440402,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1752978,
                            "linear_total": 7077888,
                            "nnz": 1761282,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 288000,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 859129,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1147129,
                            "linear_total": 7077888,
                            "nnz": 1155416,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 26420688,
                    "linear_sparsity": 68.89292399088542,
                    "linear_total": 84934656,
                    "nnz": 50358753,
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 53.75399063078199
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.5280353923367,
            "fill_rate": 0.31107076009114576,
            "speedup": 1.3799828778048573
        },
        {
            "annotate": "l=(40.0, 1.0, 1.0)",
            "cat_fun_name": "is_block_unstructured",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.86471144749291,
                    "f1": 86.87223379259328
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v8-a16-l40--2021-01-19--16-58-18/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 16,
                    "attention_block_rows": 16,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 40.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 26.916674156188964,
                    "eval_elapsed_time": 34.25446852017194
                },
                "speedup": 1.4338470191904102,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 211712,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1718621,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1930333,
                            "linear_total": 7077888,
                            "nnz": 1938429,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 345600,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1771278,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2116878,
                            "linear_total": 7077888,
                            "nnz": 2125149,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 157696,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 325955,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 483651,
                            "linear_total": 7077888,
                            "nnz": 491430,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 90368,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 164774,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 255142,
                            "linear_total": 7077888,
                            "nnz": 260749,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 278016,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1816807,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2094823,
                            "linear_total": 7077888,
                            "nnz": 2102983,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 493312,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1772769,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2266081,
                            "linear_total": 7077888,
                            "nnz": 2274689,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 304128,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1682765,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1986893,
                            "linear_total": 7077888,
                            "nnz": 1995101,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 357376,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1635131,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1992507,
                            "linear_total": 7077888,
                            "nnz": 2000891,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 278528,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1457711,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1736239,
                            "linear_total": 7077888,
                            "nnz": 1744591,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 355072,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1174807,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1529879,
                            "linear_total": 7077888,
                            "nnz": 1538375,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 183552,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 867865,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1051417,
                            "linear_total": 7077888,
                            "nnz": 1059497,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 196864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 439457,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 636321,
                            "linear_total": 7077888,
                            "nnz": 644238,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 18080164,
                    "linear_sparsity": 78.7128542676384,
                    "linear_total": 84934656,
                    "nnz": 42014844,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            8,
                            9,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 61.41646181607727
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.87223379259328,
            "fill_rate": 0.2128714573236159,
            "speedup": 1.4338470191904102
        }
    ],
    "Block/unstruct method, bs= 4x4, dl=1.0": [
        {
            "annotate": "l=(10.0, 1.0, 1.0)",
            "cat_fun_name": "is_block_unstructured",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 81.3434247871334,
                    "f1": 88.502960365548
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v8-a4-l10--2021-01-19--17-00-07/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 4,
                    "attention_block_rows": 4,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 34.458772911071776,
                    "eval_elapsed_time": 41.833797600120306
                },
                "speedup": 1.120016464456589,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 356016,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2711219,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3067235,
                            "linear_total": 7077888,
                            "nnz": 3075931,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 506400,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2753947,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3260347,
                            "linear_total": 7077888,
                            "nnz": 3269071,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 305952,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 956610,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1262562,
                            "linear_total": 7077888,
                            "nnz": 1271219,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 172864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 532866,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 705730,
                            "linear_total": 7077888,
                            "nnz": 713763,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 658880,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2827796,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3486676,
                            "linear_total": 7077888,
                            "nnz": 3495848,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 782176,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2810214,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3592390,
                            "linear_total": 7077888,
                            "nnz": 3601666,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 874272,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2770460,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3644732,
                            "linear_total": 7077888,
                            "nnz": 3654388,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 772928,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2750302,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3523230,
                            "linear_total": 7077888,
                            "nnz": 3532782,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 767984,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2610331,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3378315,
                            "linear_total": 7077888,
                            "nnz": 3387907,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 687968,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2295378,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2983346,
                            "linear_total": 7077888,
                            "nnz": 2992518,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 596368,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1868727,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2465095,
                            "linear_total": 7077888,
                            "nnz": 2474411,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 404448,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1245775,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1650223,
                            "linear_total": 7077888,
                            "nnz": 1658991,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 33019881,
                    "linear_sparsity": 61.12319451791268,
                    "linear_total": 84934656,
                    "nnz": 56967217,
                    "pruned_heads": {
                        "0": [
                            9
                        ],
                        "1": [
                            0,
                            8,
                            2,
                            5
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [],
                        "5": [
                            1
                        ],
                        "6": [
                            3
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 47.6852325727709
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.502960365548,
            "fill_rate": 0.3887680548208732,
            "speedup": 1.120016464456589
        },
        {
            "annotate": "l=(20.0, 1.0, 1.0)",
            "cat_fun_name": "is_block_unstructured",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.54872280037843,
                    "f1": 88.09731480353894
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v8-a4-l20--2021-01-19--17-00-34/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 4,
                    "attention_block_rows": 4,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 31.47156787109375,
                    "eval_elapsed_time": 38.88521202793345
                },
                "speedup": 1.2263257160702048,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 233808,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2108257,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2342065,
                            "linear_total": 7077888,
                            "nnz": 2350409,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 370912,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2165809,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2536721,
                            "linear_total": 7077888,
                            "nnz": 2545224,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 189856,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 496337,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 686193,
                            "linear_total": 7077888,
                            "nnz": 694517,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 106192,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 273404,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 379596,
                            "linear_total": 7077888,
                            "nnz": 386160,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 368864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2238488,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2607352,
                            "linear_total": 7077888,
                            "nnz": 2616080,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 528528,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2212294,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2740822,
                            "linear_total": 7077888,
                            "nnz": 2749770,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 515168,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2147598,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2662766,
                            "linear_total": 7077888,
                            "nnz": 2671906,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 456576,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2125672,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2582248,
                            "linear_total": 7077888,
                            "nnz": 2591124,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 426512,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1957790,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2384302,
                            "linear_total": 7077888,
                            "nnz": 2393358,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 424416,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1621523,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2045939,
                            "linear_total": 7077888,
                            "nnz": 2054818,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 311248,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1220304,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1531552,
                            "linear_total": 7077888,
                            "nnz": 1540408,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 249120,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 687520,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 936640,
                            "linear_total": 7077888,
                            "nnz": 945117,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 23436196,
                    "linear_sparsity": 72.40679234634212,
                    "linear_total": 84934656,
                    "nnz": 47377613,
                    "pruned_heads": {
                        "0": [
                            9,
                            2,
                            4,
                            5
                        ],
                        "1": [
                            0,
                            2,
                            5,
                            6,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            6
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            4
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 56.49166422589565
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.09731480353894,
            "fill_rate": 0.2759320765365788,
            "speedup": 1.2263257160702048
        },
        {
            "annotate": "l=(40.0, 1.0, 1.0)",
            "cat_fun_name": "is_block_unstructured",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.39451277199622,
                    "f1": 87.22039562207584
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v8-a4-l40--2021-01-19--17-01-00/checkpoint-105000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 4,
                    "attention_block_rows": 4,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 40.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 29.016168815612794,
                    "eval_elapsed_time": 36.33264479693025
                },
                "speedup": 1.3300995472773969,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 142224,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1491817,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1634041,
                            "linear_total": 7077888,
                            "nnz": 1642273,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 275888,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1552458,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1828346,
                            "linear_total": 7077888,
                            "nnz": 1836709,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 123920,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 231690,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 355610,
                            "linear_total": 7077888,
                            "nnz": 363421,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 72512,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 140404,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 212916,
                            "linear_total": 7077888,
                            "nnz": 218876,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 227744,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1597816,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1825560,
                            "linear_total": 7077888,
                            "nnz": 1833936,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 379008,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1563794,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1942802,
                            "linear_total": 7077888,
                            "nnz": 1951574,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 276192,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1485468,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1761660,
                            "linear_total": 7077888,
                            "nnz": 1770159,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 282096,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1446397,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1728493,
                            "linear_total": 7077888,
                            "nnz": 1737072,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 235856,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1268987,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1504843,
                            "linear_total": 7077888,
                            "nnz": 1513482,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 269456,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 993538,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1262994,
                            "linear_total": 7077888,
                            "nnz": 1271654,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 167520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 702743,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 870263,
                            "linear_total": 7077888,
                            "nnz": 878569,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 161424,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 328079,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 489503,
                            "linear_total": 7077888,
                            "nnz": 497608,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 15417031,
                    "linear_sparsity": 81.84836234575437,
                    "linear_total": 84934656,
                    "nnz": 39354055,
                    "pruned_heads": {
                        "0": [
                            9,
                            2,
                            4,
                            5
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            11,
                            6,
                            7
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 63.859947122862216
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.22039562207584,
            "fill_rate": 0.18151637654245634,
            "speedup": 1.3300995472773969
        },
        {
            "annotate": "l=(40.0, 1.0, 1.0)",
            "cat_fun_name": "is_block_unstructured",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.12961210974456,
                    "f1": 87.04337592394437
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v8-a4-l40--2021-01-19--17-01-00/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 4,
                    "attention_block_rows": 4,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 40.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 29.089330375671388,
                    "eval_elapsed_time": 36.40407280996442
                },
                "speedup": 1.3267542603060118,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 142224,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1491817,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1634041,
                            "linear_total": 7077888,
                            "nnz": 1642273,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 275888,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1552458,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1828346,
                            "linear_total": 7077888,
                            "nnz": 1836709,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 123920,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 231690,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 355610,
                            "linear_total": 7077888,
                            "nnz": 363421,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 72512,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 140404,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 212916,
                            "linear_total": 7077888,
                            "nnz": 218876,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 227744,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1597816,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1825560,
                            "linear_total": 7077888,
                            "nnz": 1833936,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 379008,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1563794,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1942802,
                            "linear_total": 7077888,
                            "nnz": 1951574,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 276192,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1485468,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1761660,
                            "linear_total": 7077888,
                            "nnz": 1770159,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 282096,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1446397,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1728493,
                            "linear_total": 7077888,
                            "nnz": 1737072,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 235856,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1268987,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1504843,
                            "linear_total": 7077888,
                            "nnz": 1513482,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 269456,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 993538,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1262994,
                            "linear_total": 7077888,
                            "nnz": 1271654,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 167520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 702743,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 870263,
                            "linear_total": 7077888,
                            "nnz": 878569,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 161424,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 328079,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 489503,
                            "linear_total": 7077888,
                            "nnz": 497608,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 15417031,
                    "linear_sparsity": 81.84836234575437,
                    "linear_total": 84934656,
                    "nnz": 39354055,
                    "pruned_heads": {
                        "0": [
                            9,
                            2,
                            4,
                            5
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            11,
                            6,
                            7
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 63.859947122862216
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.04337592394437,
            "fill_rate": 0.18151637654245634,
            "speedup": 1.3267542603060118
        },
        {
            "annotate": "l=(40.0, 1.0, 1.0)",
            "cat_fun_name": "is_block_unstructured",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.09176915799432,
                    "f1": 86.93076968810146
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v8-a4-l40--2021-01-19--17-01-00/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 4,
                    "attention_block_rows": 4,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 40.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 29.182387649536132,
                    "eval_elapsed_time": 36.50873678829521
                },
                "speedup": 1.3225234846739682,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 142224,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1491817,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1634041,
                            "linear_total": 7077888,
                            "nnz": 1642273,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 275888,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1552458,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1828346,
                            "linear_total": 7077888,
                            "nnz": 1836709,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 123920,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 231690,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 355610,
                            "linear_total": 7077888,
                            "nnz": 363421,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 72512,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 140404,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 212916,
                            "linear_total": 7077888,
                            "nnz": 218876,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 227744,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1597816,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1825560,
                            "linear_total": 7077888,
                            "nnz": 1833936,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 379008,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1563794,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1942802,
                            "linear_total": 7077888,
                            "nnz": 1951574,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 276192,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1485468,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1761660,
                            "linear_total": 7077888,
                            "nnz": 1770159,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 282096,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1446397,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1728493,
                            "linear_total": 7077888,
                            "nnz": 1737072,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 235856,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1268987,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1504843,
                            "linear_total": 7077888,
                            "nnz": 1513482,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 269456,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 993538,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1262994,
                            "linear_total": 7077888,
                            "nnz": 1271654,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 167520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 702743,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 870263,
                            "linear_total": 7077888,
                            "nnz": 878569,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 161424,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 328079,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 489503,
                            "linear_total": 7077888,
                            "nnz": 497608,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 15417031,
                    "linear_sparsity": 81.84836234575437,
                    "linear_total": 84934656,
                    "nnz": 39354055,
                    "pruned_heads": {
                        "0": [
                            9,
                            2,
                            4,
                            5
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            11,
                            6,
                            7
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 63.859947122862216
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.93076968810146,
            "fill_rate": 0.18151637654245634,
            "speedup": 1.3225234846739682
        }
    ],
    "Block/unstruct method, bs= 8x8, dl=1.0": [
        {
            "annotate": "l=(10.0, 1.0, 1.0)",
            "cat_fun_name": "is_block_unstructured",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 81.00283822138127,
                    "f1": 88.2671108560581
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v8-a8-l10--2021-01-19--16-58-45/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 8,
                    "attention_block_rows": 8,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 32.23066467285156,
                    "eval_elapsed_time": 39.6229472043924
                },
                "speedup": 1.1974432856757005,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 404736,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2826896,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3231632,
                            "linear_total": 7077888,
                            "nnz": 3240008,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 543040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2868676,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3411716,
                            "linear_total": 7077888,
                            "nnz": 3420292,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 322624,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1081551,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1404175,
                            "linear_total": 7077888,
                            "nnz": 1412684,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 172288,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 591605,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 763893,
                            "linear_total": 7077888,
                            "nnz": 771824,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 614464,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2943501,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3557965,
                            "linear_total": 7077888,
                            "nnz": 3566861,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 790144,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2916630,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3706774,
                            "linear_total": 7077888,
                            "nnz": 3715734,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 816832,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2876748,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3693580,
                            "linear_total": 7077888,
                            "nnz": 3702756,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 785920,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2855585,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3641505,
                            "linear_total": 7077888,
                            "nnz": 3650569,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 759424,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2724738,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3484162,
                            "linear_total": 7077888,
                            "nnz": 3493306,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 687040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2427854,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3114894,
                            "linear_total": 7077888,
                            "nnz": 3123814,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 603648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2013418,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2617066,
                            "linear_total": 7077888,
                            "nnz": 2626058,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 379328,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1394031,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1773359,
                            "linear_total": 7077888,
                            "nnz": 1781871,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 34400721,
                    "linear_sparsity": 59.49742705733687,
                    "linear_total": 84934656,
                    "nnz": 58344499,
                    "pruned_heads": {
                        "0": [
                            9,
                            2,
                            4,
                            5
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            10,
                            4,
                            6
                        ],
                        "4": [
                            6
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7
                        ],
                        "6": [
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            4
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 46.42043166961797
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.2671108560581,
            "fill_rate": 0.4050257294266313,
            "speedup": 1.1974432856757005
        },
        {
            "annotate": "l=(10.0, 1.0, 1.0)",
            "cat_fun_name": "is_block_unstructured",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 81.01229895931883,
                    "f1": 88.16022239737082
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v8-a8-l10--2021-01-19--16-58-45/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 8,
                    "attention_block_rows": 8,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 32.31462289428711,
                    "eval_elapsed_time": 39.686994375661016
                },
                "speedup": 1.1943321489972945,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 404736,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2826896,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3231632,
                            "linear_total": 7077888,
                            "nnz": 3240008,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 543040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2868676,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3411716,
                            "linear_total": 7077888,
                            "nnz": 3420292,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 322624,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1081551,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1404175,
                            "linear_total": 7077888,
                            "nnz": 1412684,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 172288,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 591605,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 763893,
                            "linear_total": 7077888,
                            "nnz": 771824,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 614464,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2943501,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3557965,
                            "linear_total": 7077888,
                            "nnz": 3566861,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 790144,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2916630,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3706774,
                            "linear_total": 7077888,
                            "nnz": 3715734,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 816832,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2876748,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3693580,
                            "linear_total": 7077888,
                            "nnz": 3702756,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 785920,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2855585,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3641505,
                            "linear_total": 7077888,
                            "nnz": 3650569,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 759424,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2724738,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3484162,
                            "linear_total": 7077888,
                            "nnz": 3493306,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 687040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2427854,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3114894,
                            "linear_total": 7077888,
                            "nnz": 3123814,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 603648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2013418,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2617066,
                            "linear_total": 7077888,
                            "nnz": 2626058,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 379328,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1394031,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1773359,
                            "linear_total": 7077888,
                            "nnz": 1781871,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 34400721,
                    "linear_sparsity": 59.49742705733687,
                    "linear_total": 84934656,
                    "nnz": 58344499,
                    "pruned_heads": {
                        "0": [
                            9,
                            2,
                            4,
                            5
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            10,
                            4,
                            6
                        ],
                        "4": [
                            6
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7
                        ],
                        "6": [
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            4
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 46.42043166961797
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.16022239737082,
            "fill_rate": 0.4050257294266313,
            "speedup": 1.1943321489972945
        },
        {
            "annotate": "l=(20.0, 1.0, 1.0)",
            "cat_fun_name": "is_block_unstructured",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.15137180700094,
                    "f1": 87.62280270760408
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v8-a8-l20--2021-01-19--16-59-13/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 8,
                    "attention_block_rows": 8,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 28.9650231628418,
                    "eval_elapsed_time": 36.364678455051035
                },
                "speedup": 1.3324482010041157,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 278464,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2254373,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2532837,
                            "linear_total": 7077888,
                            "nnz": 2541085,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 411200,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2313203,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2724403,
                            "linear_total": 7077888,
                            "nnz": 2732755,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 207872,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 587562,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 795434,
                            "linear_total": 7077888,
                            "nnz": 803713,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 115648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 304918,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 420566,
                            "linear_total": 7077888,
                            "nnz": 427027,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 388544,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2383637,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2772181,
                            "linear_total": 7077888,
                            "nnz": 2780605,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 616064,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2346825,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2962889,
                            "linear_total": 7077888,
                            "nnz": 2971721,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 475392,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2281407,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2756799,
                            "linear_total": 7077888,
                            "nnz": 2765383,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 485760,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2255524,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2741284,
                            "linear_total": 7077888,
                            "nnz": 2749908,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 436416,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2089830,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2526246,
                            "linear_total": 7077888,
                            "nnz": 2534870,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 473664,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1759353,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2233017,
                            "linear_total": 7077888,
                            "nnz": 2241713,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 292096,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1360596,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1652692,
                            "linear_total": 7077888,
                            "nnz": 1661036,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 260864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 795671,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1056535,
                            "linear_total": 7077888,
                            "nnz": 1064961,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 25174883,
                    "linear_sparsity": 70.35970452391072,
                    "linear_total": 84934656,
                    "nnz": 49113499,
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            10,
                            4,
                            6
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 54.89754611459343
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.62280270760408,
            "fill_rate": 0.29640295476089273,
            "speedup": 1.3324482010041157
        },
        {
            "annotate": "l=(40.0, 1.0, 1.0)",
            "cat_fun_name": "is_block_unstructured",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.59981078524125,
                    "f1": 86.70965342219107
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v8-a8-l40--2021-01-19--16-59-40/checkpoint-105000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 8,
                    "attention_block_rows": 8,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 40.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 27.427432876586913,
                    "eval_elapsed_time": 34.77788851317018
                },
                "speedup": 1.407145655192423,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 180736,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1630123,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1810859,
                            "linear_total": 7077888,
                            "nnz": 1818979,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 305920,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1692103,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1998023,
                            "linear_total": 7077888,
                            "nnz": 2006245,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 135616,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 281100,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 416716,
                            "linear_total": 7077888,
                            "nnz": 424480,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 81536,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 153912,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 235448,
                            "linear_total": 7077888,
                            "nnz": 241254,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 263936,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1737263,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2001199,
                            "linear_total": 7077888,
                            "nnz": 2009399,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 442496,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1689622,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2132118,
                            "linear_total": 7077888,
                            "nnz": 2140797,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 306304,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1612927,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1919231,
                            "linear_total": 7077888,
                            "nnz": 1927591,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 312128,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1572769,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1884897,
                            "linear_total": 7077888,
                            "nnz": 1893352,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 258304,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1387454,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1645758,
                            "linear_total": 7077888,
                            "nnz": 1654182,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 305856,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1098709,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1404565,
                            "linear_total": 7077888,
                            "nnz": 1413067,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 172480,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 791310,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 963790,
                            "linear_total": 7077888,
                            "nnz": 971964,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 170944,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 393127,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 564071,
                            "linear_total": 7077888,
                            "nnz": 572153,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 16976675,
                    "linear_sparsity": 80.01207540064682,
                    "linear_total": 84934656,
                    "nnz": 40912185,
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            10,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            8,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 62.42906787574385
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.70965342219107,
            "fill_rate": 0.1998792459935318,
            "speedup": 1.407145655192423
        },
        {
            "annotate": "l=(40.0, 1.0, 1.0)",
            "cat_fun_name": "is_block_unstructured",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.78902554399244,
                    "f1": 86.80367154149816
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v8-a8-l40--2021-01-19--16-59-40/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 8,
                    "attention_block_rows": 8,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 40.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 27.48367044067383,
                    "eval_elapsed_time": 34.82450146274641
                },
                "speedup": 1.404266329298368,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 180736,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1630123,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1810859,
                            "linear_total": 7077888,
                            "nnz": 1818979,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 305920,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1692103,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1998023,
                            "linear_total": 7077888,
                            "nnz": 2006245,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 135616,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 281100,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 416716,
                            "linear_total": 7077888,
                            "nnz": 424480,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 81536,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 153912,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 235448,
                            "linear_total": 7077888,
                            "nnz": 241254,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 263936,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1737263,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2001199,
                            "linear_total": 7077888,
                            "nnz": 2009399,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 442496,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1689622,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2132118,
                            "linear_total": 7077888,
                            "nnz": 2140797,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 306304,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1612927,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1919231,
                            "linear_total": 7077888,
                            "nnz": 1927591,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 312128,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1572769,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1884897,
                            "linear_total": 7077888,
                            "nnz": 1893352,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 258304,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1387454,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1645758,
                            "linear_total": 7077888,
                            "nnz": 1654182,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 305856,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1098709,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1404565,
                            "linear_total": 7077888,
                            "nnz": 1413067,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 172480,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 791310,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 963790,
                            "linear_total": 7077888,
                            "nnz": 971964,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 170944,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 393127,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 564071,
                            "linear_total": 7077888,
                            "nnz": 572153,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 16976675,
                    "linear_sparsity": 80.01207540064682,
                    "linear_total": 84934656,
                    "nnz": 40912185,
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            10,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            8,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 62.42906787574385
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.80367154149816,
            "fill_rate": 0.1998792459935318,
            "speedup": 1.404266329298368
        },
        {
            "annotate": "l=(40.0, 1.0, 1.0)",
            "cat_fun_name": "is_block_unstructured",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.6092715231788,
                    "f1": 86.70267601348202
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v8-a8-l40--2021-01-19--16-59-40/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 8,
                    "attention_block_rows": 8,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 40.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 27.478721130371095,
                    "eval_elapsed_time": 34.80613293591887
                },
                "speedup": 1.4045192577290035,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 180736,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1630123,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1810859,
                            "linear_total": 7077888,
                            "nnz": 1818979,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 305920,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1692103,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1998023,
                            "linear_total": 7077888,
                            "nnz": 2006245,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 135616,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 281100,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 416716,
                            "linear_total": 7077888,
                            "nnz": 424480,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 81536,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 153912,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 235448,
                            "linear_total": 7077888,
                            "nnz": 241254,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 263936,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1737263,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2001199,
                            "linear_total": 7077888,
                            "nnz": 2009399,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 442496,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1689622,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2132118,
                            "linear_total": 7077888,
                            "nnz": 2140797,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 306304,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1612927,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1919231,
                            "linear_total": 7077888,
                            "nnz": 1927591,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 312128,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1572769,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1884897,
                            "linear_total": 7077888,
                            "nnz": 1893352,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 258304,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1387454,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1645758,
                            "linear_total": 7077888,
                            "nnz": 1654182,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 305856,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1098709,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1404565,
                            "linear_total": 7077888,
                            "nnz": 1413067,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 172480,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 791310,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 963790,
                            "linear_total": 7077888,
                            "nnz": 971964,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 170944,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 393127,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 564071,
                            "linear_total": 7077888,
                            "nnz": 572153,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 16976675,
                    "linear_sparsity": 80.01207540064682,
                    "linear_total": 84934656,
                    "nnz": 40912185,
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            10,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            8,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 62.42906787574385
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.70267601348202,
            "fill_rate": 0.1998792459935318,
            "speedup": 1.4045192577290035
        }
    ],
    "Full block method, bs= 16x16, dl=0.25": [
        {
            "annotate": "57",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.93661305581836,
                    "f1": 88.34112193061533
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v9-a16-l10--2021-01-20--18-58-11/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 16,
                    "attention_block_rows": 16,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 16,
                    "dense_block_rows": 16,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 30.13610975646973,
                    "eval_elapsed_time": 37.54532916797325
                },
                "speedup": 1.2806693802635063,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 517888,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4068608,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4586496,
                            "linear_total": 7077888,
                            "nnz": 4594896,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 641536,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4202752,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4844288,
                            "linear_total": 7077888,
                            "nnz": 4852816,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 415488,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1090304,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1505792,
                            "linear_total": 7077888,
                            "nnz": 1513344,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 254720,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 947200,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1201920,
                            "linear_total": 7077888,
                            "nnz": 1208720,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 841472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4313856,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5155328,
                            "linear_total": 7077888,
                            "nnz": 5164240,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1072896,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4336128,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5409024,
                            "linear_total": 7077888,
                            "nnz": 5418160,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1068800,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4317184,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5385984,
                            "linear_total": 7077888,
                            "nnz": 5395264,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 961792,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4311040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5272832,
                            "linear_total": 7077888,
                            "nnz": 5281824,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 986880,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4141568,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5128448,
                            "linear_total": 7077888,
                            "nnz": 5137632,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 905472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3820032,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4725504,
                            "linear_total": 7077888,
                            "nnz": 4734512,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 756224,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3085568,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3841792,
                            "linear_total": 7077888,
                            "nnz": 3850720,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 463360,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1416448,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1879808,
                            "linear_total": 7077888,
                            "nnz": 1887632,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 48937216,
                    "linear_sparsity": 42.38251109182099,
                    "linear_total": 84934656,
                    "nnz": 72878482,
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            1,
                            2
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            2,
                            3,
                            7
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            8,
                            4
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 33.07342297799975
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.34112193061533,
            "fill_rate": 0.5761748890817902,
            "speedup": 1.2806693802635063
        },
        {
            "annotate": "46",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.02838221381268,
                    "f1": 87.51569063636161
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v9-a16-l20--2021-01-20--18-58-39/checkpoint-105000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 16,
                    "attention_block_rows": 16,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 16,
                    "dense_block_rows": 16,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 26.3544778213501,
                    "eval_elapsed_time": 33.69302155217156
                },
                "speedup": 1.4644339860190774,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 417024,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3447808,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3864832,
                            "linear_total": 7077888,
                            "nnz": 3873088,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 542720,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3703296,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4246016,
                            "linear_total": 7077888,
                            "nnz": 4254480,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 273408,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 647424,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 920832,
                            "linear_total": 7077888,
                            "nnz": 927232,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 166400,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 608512,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 774912,
                            "linear_total": 7077888,
                            "nnz": 781008,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 555776,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3981824,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4537600,
                            "linear_total": 7077888,
                            "nnz": 4546144,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 810240,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4014336,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4824576,
                            "linear_total": 7077888,
                            "nnz": 4833424,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 764160,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3940608,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4704768,
                            "linear_total": 7077888,
                            "nnz": 4713616,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 685824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3904256,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4590080,
                            "linear_total": 7077888,
                            "nnz": 4598784,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 647680,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3571456,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4219136,
                            "linear_total": 7077888,
                            "nnz": 4227936,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 684288,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2956288,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3640576,
                            "linear_total": 7077888,
                            "nnz": 3649248,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 427264,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1932800,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2360064,
                            "linear_total": 7077888,
                            "nnz": 2368032,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 350976,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 682496,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1033472,
                            "linear_total": 7077888,
                            "nnz": 1039984,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 39716864,
                    "linear_sparsity": 53.238329475308646,
                    "linear_total": 84934656,
                    "nnz": 63651698,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            7,
                            8
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 41.546665739029805
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.51569063636161,
            "fill_rate": 0.46761670524691357,
            "speedup": 1.4644339860190774
        },
        {
            "annotate": "46",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.55534531693472,
                    "f1": 87.439750439335
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v9-a16-l20--2021-01-20--18-58-39/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 16,
                    "attention_block_rows": 16,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 16,
                    "dense_block_rows": 16,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 26.32847610473633,
                    "eval_elapsed_time": 33.60846929671243
                },
                "speedup": 1.4658802450943298,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 417024,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3447808,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3864832,
                            "linear_total": 7077888,
                            "nnz": 3873088,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 542720,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3703296,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4246016,
                            "linear_total": 7077888,
                            "nnz": 4254480,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 273408,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 647424,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 920832,
                            "linear_total": 7077888,
                            "nnz": 927232,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 166400,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 608512,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 774912,
                            "linear_total": 7077888,
                            "nnz": 781008,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 555776,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3981824,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4537600,
                            "linear_total": 7077888,
                            "nnz": 4546144,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 810240,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4014336,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4824576,
                            "linear_total": 7077888,
                            "nnz": 4833424,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 764160,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3940608,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4704768,
                            "linear_total": 7077888,
                            "nnz": 4713616,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 685824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3904256,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4590080,
                            "linear_total": 7077888,
                            "nnz": 4598784,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 647680,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3571456,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4219136,
                            "linear_total": 7077888,
                            "nnz": 4227936,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 684288,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2956288,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3640576,
                            "linear_total": 7077888,
                            "nnz": 3649248,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 427264,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1932800,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2360064,
                            "linear_total": 7077888,
                            "nnz": 2368032,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 350976,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 682496,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1033472,
                            "linear_total": 7077888,
                            "nnz": 1039984,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 39716864,
                    "linear_sparsity": 53.238329475308646,
                    "linear_total": 84934656,
                    "nnz": 63651698,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            7,
                            8
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 41.546665739029805
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.439750439335,
            "fill_rate": 0.46761670524691357,
            "speedup": 1.4658802450943298
        },
        {
            "annotate": "46",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.67833491012298,
                    "f1": 87.29496050765553
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v9-a16-l20--2021-01-20--18-58-39/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 16,
                    "attention_block_rows": 16,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 16,
                    "dense_block_rows": 16,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 26.343981628417968,
                    "eval_elapsed_time": 33.70636031124741
                },
                "speedup": 1.4650174582470206,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 417024,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3447808,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3864832,
                            "linear_total": 7077888,
                            "nnz": 3873088,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 542720,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3703296,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4246016,
                            "linear_total": 7077888,
                            "nnz": 4254480,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 273408,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 647424,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 920832,
                            "linear_total": 7077888,
                            "nnz": 927232,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 166400,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 608512,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 774912,
                            "linear_total": 7077888,
                            "nnz": 781008,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 555776,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3981824,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4537600,
                            "linear_total": 7077888,
                            "nnz": 4546144,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 810240,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4014336,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4824576,
                            "linear_total": 7077888,
                            "nnz": 4833424,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 764160,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3940608,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4704768,
                            "linear_total": 7077888,
                            "nnz": 4713616,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 685824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3904256,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4590080,
                            "linear_total": 7077888,
                            "nnz": 4598784,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 647680,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3571456,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4219136,
                            "linear_total": 7077888,
                            "nnz": 4227936,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 684288,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2956288,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3640576,
                            "linear_total": 7077888,
                            "nnz": 3649248,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 427264,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1932800,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2360064,
                            "linear_total": 7077888,
                            "nnz": 2368032,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 350976,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 682496,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1033472,
                            "linear_total": 7077888,
                            "nnz": 1039984,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 39716864,
                    "linear_sparsity": 53.238329475308646,
                    "linear_total": 84934656,
                    "nnz": 63651698,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            7,
                            8
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 41.546665739029805
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 79.91485335856197,
                    "f1": 87.42973403288855
                }
            },
            "f1": 87.29496050765553,
            "fill_rate": 0.46761670524691357,
            "speedup": 1.4650174582470206
        },
        {
            "annotate": "34",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.11731315042573,
                    "f1": 86.14927876930865
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v9-a16-l40--2021-01-20--18-59-08/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 16,
                    "attention_block_rows": 16,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 16,
                    "dense_block_rows": 16,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 40.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 23.35162329864502,
                    "eval_elapsed_time": 30.60480569722131
                },
                "speedup": 1.6527498971607057,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 331008,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2354688,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2685696,
                            "linear_total": 7077888,
                            "nnz": 2693408,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 432384,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2826240,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3258624,
                            "linear_total": 7077888,
                            "nnz": 3266704,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 203008,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 415744,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 618752,
                            "linear_total": 7077888,
                            "nnz": 624640,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 112128,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 423168,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 535296,
                            "linear_total": 7077888,
                            "nnz": 540768,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 423936,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3302144,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3726080,
                            "linear_total": 7077888,
                            "nnz": 3734288,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 669440,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3248128,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3917568,
                            "linear_total": 7077888,
                            "nnz": 3926240,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 453632,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3193600,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3647232,
                            "linear_total": 7077888,
                            "nnz": 3655504,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 473856,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3119616,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3593472,
                            "linear_total": 7077888,
                            "nnz": 3601824,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 445952,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2493696,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2939648,
                            "linear_total": 7077888,
                            "nnz": 2947744,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 490752,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1891072,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2381824,
                            "linear_total": 7077888,
                            "nnz": 2389520,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 275712,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1108736,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1384448,
                            "linear_total": 7077888,
                            "nnz": 1391248,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 258304,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 348928,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 607232,
                            "linear_total": 7077888,
                            "nnz": 612928,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 29295872,
                    "linear_sparsity": 65.5077522183642,
                    "linear_total": 84934656,
                    "nnz": 53223538,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            8,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 51.12316945157615
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.14927876930865,
            "fill_rate": 0.344922477816358,
            "speedup": 1.6527498971607057
        }
    ],
    "Full block method, bs= 16x16, dl=1.0": [
        {
            "annotate": "35",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.00946073793756,
                    "f1": 87.65780769915727
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v11-a16-l10-dl1--2021-01-24--15-45-00/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 16,
                    "attention_block_rows": 16,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 16,
                    "dense_block_rows": 16,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 26.317300163269042,
                    "eval_elapsed_time": 33.56822411296889
                },
                "speedup": 1.4665027478478643,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 720896,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1657600,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2378496,
                            "linear_total": 7077888,
                            "nnz": 2386160,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 719872,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2046464,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2766336,
                            "linear_total": 7077888,
                            "nnz": 2774128,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 450560,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 272128,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 722688,
                            "linear_total": 7077888,
                            "nnz": 728816,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 307456,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 311808,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 619264,
                            "linear_total": 7077888,
                            "nnz": 625232,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1058304,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2721792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3780096,
                            "linear_total": 7077888,
                            "nnz": 3788768,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1227776,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2707200,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3934976,
                            "linear_total": 7077888,
                            "nnz": 3943840,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1367808,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2789888,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4157696,
                            "linear_total": 7077888,
                            "nnz": 4166848,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1258240,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2672384,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3930624,
                            "linear_total": 7077888,
                            "nnz": 3939824,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1130496,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2136064,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3266560,
                            "linear_total": 7077888,
                            "nnz": 3275008,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 988928,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1491200,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2480128,
                            "linear_total": 7077888,
                            "nnz": 2487824,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 888576,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 653568,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1542144,
                            "linear_total": 7077888,
                            "nnz": 1549312,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 567296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 249088,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 816384,
                            "linear_total": 7077888,
                            "nnz": 822432,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 30395392,
                    "linear_sparsity": 64.21320408950618,
                    "linear_total": 84934656,
                    "nnz": 54326914,
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            5,
                            6,
                            7
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            5,
                            6,
                            8,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            2
                        ],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            2,
                            3,
                            7
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            8,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 50.10990494850615
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.65780769915727,
            "fill_rate": 0.3578679591049382,
            "speedup": 1.4665027478478643
        },
        {
            "annotate": "23",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.62819299905392,
                    "f1": 86.57822332702295
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v11-a16-l20-dl1--2021-01-24--15-45-27/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 16,
                    "attention_block_rows": 16,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 16,
                    "dense_block_rows": 16,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 21.08596396636963,
                    "eval_elapsed_time": 28.310240568593144
                },
                "speedup": 1.8303357184393354,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 484864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 821248,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1306112,
                            "linear_total": 7077888,
                            "nnz": 1312512,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 604160,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1247488,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1851648,
                            "linear_total": 7077888,
                            "nnz": 1858512,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 343296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 200704,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 544000,
                            "linear_total": 7077888,
                            "nnz": 549840,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 215296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 216320,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 431616,
                            "linear_total": 7077888,
                            "nnz": 437072,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 813312,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1803264,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2616576,
                            "linear_total": 7077888,
                            "nnz": 2624192,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1050880,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1707520,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2758400,
                            "linear_total": 7077888,
                            "nnz": 2766304,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1007104,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1634816,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2641920,
                            "linear_total": 7077888,
                            "nnz": 2649840,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 769792,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1574400,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2344192,
                            "linear_total": 7077888,
                            "nnz": 2351712,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 749056,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1194496,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1943552,
                            "linear_total": 7077888,
                            "nnz": 1950752,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 765440,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 861440,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1626880,
                            "linear_total": 7077888,
                            "nnz": 1633760,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 645888,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 319488,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 965376,
                            "linear_total": 7077888,
                            "nnz": 971712,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 368128,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 145920,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 514048,
                            "linear_total": 7077888,
                            "nnz": 519680,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 19544320,
                    "linear_sparsity": 76.98899257330247,
                    "linear_total": 84934656,
                    "nnz": 43464610,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            6,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            11,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            0,
                            2
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            4
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 60.08509660099393
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 78.7511825922422,
                    "f1": 86.70333537174074
                }
            },
            "f1": 86.57822332702295,
            "fill_rate": 0.23011007426697527,
            "speedup": 1.8303357184393354
        }
    ],
    "Full block method, bs= 32x32, dl=0.25": [
        {
            "annotate": "57",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.81362346263009,
                    "f1": 88.10463591853348
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_stl50_est5000_rn-output__squad_test3_dpm-sigmoied_threshold_apme-sigmoied_threshold_aowd0_bm1_dbr32_dbc32_abr32_abc32_it0_fw10_r-l1_rfl10_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-105000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 32,
                    "dense_block_rows": 32,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 27.2810027923584,
                    "eval_elapsed_time": 34.61669071530923
                },
                "speedup": 1.4146984734806616,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 644096,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4032512,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4676608,
                            "linear_total": 7077888,
                            "nnz": 4685024,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 583680,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4172800,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4756480,
                            "linear_total": 7077888,
                            "nnz": 4764896,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 445440,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 890880,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1336320,
                            "linear_total": 7077888,
                            "nnz": 1342880,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 272384,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 875520,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1147904,
                            "linear_total": 7077888,
                            "nnz": 1154112,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 789504,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4324352,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5113856,
                            "linear_total": 7077888,
                            "nnz": 5122528,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1028096,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4392960,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5421056,
                            "linear_total": 7077888,
                            "nnz": 5430016,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1067008,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4359168,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5426176,
                            "linear_total": 7077888,
                            "nnz": 5435200,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 943104,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4333568,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5276672,
                            "linear_total": 7077888,
                            "nnz": 5285504,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1003520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4161536,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5165056,
                            "linear_total": 7077888,
                            "nnz": 5174112,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 908288,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3889152,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4797440,
                            "linear_total": 7077888,
                            "nnz": 4806272,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 868352,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3021824,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3890176,
                            "linear_total": 7077888,
                            "nnz": 3899008,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 520192,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1206272,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1726464,
                            "linear_total": 7077888,
                            "nnz": 1733312,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 48734208,
                    "linear_sparsity": 42.62152777777778,
                    "linear_total": 84934656,
                    "nnz": 72671586,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            4,
                            7,
                            8
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            1,
                            2,
                            11,
                            6
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 33.26342201062975
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.10463591853348,
            "fill_rate": 0.5737847222222222,
            "speedup": 1.4146984734806616
        },
        {
            "annotate": "57",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.53926206244087,
                    "f1": 87.95145431777735
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_stl50_est5000_rn-output__squad_test3_dpm-sigmoied_threshold_apme-sigmoied_threshold_aowd0_bm1_dbr32_dbc32_abr32_abc32_it0_fw10_r-l1_rfl10_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 32,
                    "dense_block_rows": 32,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 27.25869842529297,
                    "eval_elapsed_time": 34.5833341376856
                },
                "speedup": 1.4158560472410484,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 614400,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4140032,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4754432,
                            "linear_total": 7077888,
                            "nnz": 4762848,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 596992,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4246528,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4843520,
                            "linear_total": 7077888,
                            "nnz": 4851936,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 450560,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 881664,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1332224,
                            "linear_total": 7077888,
                            "nnz": 1338784,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 266240,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 863232,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1129472,
                            "linear_total": 7077888,
                            "nnz": 1135648,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 788480,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4357120,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5145600,
                            "linear_total": 7077888,
                            "nnz": 5154272,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1061888,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4426752,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5488640,
                            "linear_total": 7077888,
                            "nnz": 5497600,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1048576,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4414464,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5463040,
                            "linear_total": 7077888,
                            "nnz": 5472064,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 918528,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4399104,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5317632,
                            "linear_total": 7077888,
                            "nnz": 5326464,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 998400,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4232192,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5230592,
                            "linear_total": 7077888,
                            "nnz": 5239648,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 899072,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3939328,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4838400,
                            "linear_total": 7077888,
                            "nnz": 4847232,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 819200,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3028992,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3848192,
                            "linear_total": 7077888,
                            "nnz": 3857024,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 516096,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1104896,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1620992,
                            "linear_total": 7077888,
                            "nnz": 1627840,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 49012736,
                    "linear_sparsity": 42.29359567901234,
                    "linear_total": 84934656,
                    "nnz": 72950082,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            4,
                            7,
                            8
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            1,
                            2,
                            11,
                            6
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 33.00767047076757
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.95145431777735,
            "fill_rate": 0.5770640432098766,
            "speedup": 1.4158560472410484
        },
        {
            "annotate": "57",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.72847682119205,
                    "f1": 88.08831525592305
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_stl50_est5000_rn-output__squad_test3_dpm-sigmoied_threshold_apme-sigmoied_threshold_aowd0_bm1_dbr32_dbc32_abr32_abc32_it0_fw10_r-l1_rfl10_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-95000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 32,
                    "dense_block_rows": 32,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 27.293812591552737,
                    "eval_elapsed_time": 34.635603360366076
                },
                "speedup": 1.4140345133503194,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 621568,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4035584,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4657152,
                            "linear_total": 7077888,
                            "nnz": 4665568,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 604160,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4155392,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4759552,
                            "linear_total": 7077888,
                            "nnz": 4767968,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 486400,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 957440,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1443840,
                            "linear_total": 7077888,
                            "nnz": 1450400,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 286720,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 891904,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1178624,
                            "linear_total": 7077888,
                            "nnz": 1184800,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 781312,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4284416,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5065728,
                            "linear_total": 7077888,
                            "nnz": 5074400,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1068032,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4340736,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5408768,
                            "linear_total": 7077888,
                            "nnz": 5417792,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1087488,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4237312,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5324800,
                            "linear_total": 7077888,
                            "nnz": 5333888,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 908288,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4281344,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5189632,
                            "linear_total": 7077888,
                            "nnz": 5198464,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1019904,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4149248,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5169152,
                            "linear_total": 7077888,
                            "nnz": 5178208,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 921600,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3827712,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4749312,
                            "linear_total": 7077888,
                            "nnz": 4758144,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 851968,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3080192,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3932160,
                            "linear_total": 7077888,
                            "nnz": 3941088,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 529408,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1278976,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1808384,
                            "linear_total": 7077888,
                            "nnz": 1815360,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 48687104,
                    "linear_sparsity": 42.67698688271605,
                    "linear_total": 84934656,
                    "nnz": 72624802,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            4,
                            7,
                            8
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            1,
                            2,
                            11,
                            6
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 33.306385213120684
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.08831525592305,
            "fill_rate": 0.5732301311728395,
            "speedup": 1.4140345133503194
        },
        {
            "annotate": "47",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.29990539262063,
                    "f1": 87.09851869948527
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_stl50_est5000_rn-output__squad_test3_dpm-sigmoied_threshold_apme-sigmoied_threshold_aowd0_bm1_dbr32_dbc32_abr32_abc32_it0_fw10_r-l1_rfl20_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-105000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 32,
                    "dense_block_rows": 32,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20
                },
                "speed": {
                    "cuda_eval_elapsed_time": 24.736273986816407,
                    "eval_elapsed_time": 32.05209435708821
                },
                "speedup": 1.5602346992898202,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 527360,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3380224,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3907584,
                            "linear_total": 7077888,
                            "nnz": 3915840,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3661824,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4186112,
                            "linear_total": 7077888,
                            "nnz": 4194400,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 307200,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 595968,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 903168,
                            "linear_total": 7077888,
                            "nnz": 908864,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 207872,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 615424,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 823296,
                            "linear_total": 7077888,
                            "nnz": 828864,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 598016,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4031488,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4629504,
                            "linear_total": 7077888,
                            "nnz": 4638016,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 930816,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4107264,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5038080,
                            "linear_total": 7077888,
                            "nnz": 5047008,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 824320,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3997696,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4822016,
                            "linear_total": 7077888,
                            "nnz": 4830816,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 746496,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4027392,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4773888,
                            "linear_total": 7077888,
                            "nnz": 4782592,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 670720,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3737600,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4408320,
                            "linear_total": 7077888,
                            "nnz": 4416832,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 794624,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2995200,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3789824,
                            "linear_total": 7077888,
                            "nnz": 3798144,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 419840,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1756160,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2176000,
                            "linear_total": 7077888,
                            "nnz": 2183232,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 411648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 600064,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1011712,
                            "linear_total": 7077888,
                            "nnz": 1017600,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 40469504,
                    "linear_sparsity": 52.35218942901234,
                    "linear_total": 84934656,
                    "nnz": 64400930,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 40.85862268737366
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.09851869948527,
            "fill_rate": 0.4764781057098766,
            "speedup": 1.5602346992898202
        },
        {
            "annotate": "47",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.22421948912014,
                    "f1": 87.0664817371684
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_stl50_est5000_rn-output__squad_test3_dpm-sigmoied_threshold_apme-sigmoied_threshold_aowd0_bm1_dbr32_dbc32_abr32_abc32_it0_fw10_r-l1_rfl20_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 32,
                    "dense_block_rows": 32,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20
                },
                "speed": {
                    "cuda_eval_elapsed_time": 24.675214500427245,
                    "eval_elapsed_time": 31.986000607255846
                },
                "speedup": 1.5640955422982379,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 501760,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3380224,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3881984,
                            "linear_total": 7077888,
                            "nnz": 3890208,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 528384,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3656704,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4185088,
                            "linear_total": 7077888,
                            "nnz": 4193376,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 313344,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 561152,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 874496,
                            "linear_total": 7077888,
                            "nnz": 880224,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 200704,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 617472,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 818176,
                            "linear_total": 7077888,
                            "nnz": 823744,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 581632,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4121600,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4703232,
                            "linear_total": 7077888,
                            "nnz": 4711744,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 916480,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4144128,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5060608,
                            "linear_total": 7077888,
                            "nnz": 5069536,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 833536,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4060160,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4893696,
                            "linear_total": 7077888,
                            "nnz": 4902496,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 741376,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4076544,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4817920,
                            "linear_total": 7077888,
                            "nnz": 4826624,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 644096,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3815424,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4459520,
                            "linear_total": 7077888,
                            "nnz": 4468032,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 757760,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2962432,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3720192,
                            "linear_total": 7077888,
                            "nnz": 3728512,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 380928,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1689600,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2070528,
                            "linear_total": 7077888,
                            "nnz": 2077792,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 395264,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 571392,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 966656,
                            "linear_total": 7077888,
                            "nnz": 972576,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 40452096,
                    "linear_sparsity": 52.37268518518518,
                    "linear_total": 84934656,
                    "nnz": 64383586,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 40.874550222086434
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.0664817371684,
            "fill_rate": 0.47627314814814814,
            "speedup": 1.5640955422982379
        },
        {
            "annotate": "51",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.67833491012298,
                    "f1": 87.14623278516426
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_stl50_est5000_rn-output__squad_test3_dpm-sigmoied_threshold_apme-sigmoied_threshold_aowd0_bm1_dbr32_dbc32_abr32_abc32_it0_fw10_r-l1_rfl20_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-65000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 32,
                    "dense_block_rows": 32,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20
                },
                "speed": {
                    "cuda_eval_elapsed_time": 25.61453672027588,
                    "eval_elapsed_time": 32.96429116372019
                },
                "speedup": 1.5067378897710322,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 571392,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3765248,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4336640,
                            "linear_total": 7077888,
                            "nnz": 4345056,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 599040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3852288,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4451328,
                            "linear_total": 7077888,
                            "nnz": 4459744,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 374784,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 672768,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1047552,
                            "linear_total": 7077888,
                            "nnz": 1053600,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 235520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 706560,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 942080,
                            "linear_total": 7077888,
                            "nnz": 947776,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 695296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4087808,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4783104,
                            "linear_total": 7077888,
                            "nnz": 4791712,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 996352,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4050944,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5047296,
                            "linear_total": 7077888,
                            "nnz": 5056256,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 923648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4109312,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5032960,
                            "linear_total": 7077888,
                            "nnz": 5041824,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 865280,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4041728,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4907008,
                            "linear_total": 7077888,
                            "nnz": 4915776,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 778240,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3858432,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4636672,
                            "linear_total": 7077888,
                            "nnz": 4645440,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 883712,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3359744,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4243456,
                            "linear_total": 7077888,
                            "nnz": 4252160,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 513024,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2305024,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2818048,
                            "linear_total": 7077888,
                            "nnz": 2826048,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 462848,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 826368,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1289216,
                            "linear_total": 7077888,
                            "nnz": 1295424,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 43535360,
                    "linear_sparsity": 48.742525077160494,
                    "linear_total": 84934656,
                    "nnz": 67469538,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 38.04062450702838
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.14623278516426,
            "fill_rate": 0.5125747492283951,
            "speedup": 1.5067378897710322
        },
        {
            "annotate": "34",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.70104068117313,
                    "f1": 85.88451743537976
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_stl50_est5000_rn-output__squad_test3_dpm-sigmoied_threshold_apme-sigmoied_threshold_aowd0_bm1_dbr32_dbc32_abr32_abc32_it0_fw10_r-l1_rfl40_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 32,
                    "dense_block_rows": 32,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 40
                },
                "speed": {
                    "cuda_eval_elapsed_time": 20.68525614929199,
                    "eval_elapsed_time": 27.97377561684698
                },
                "speedup": 1.8657923656745288,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 413696,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2119680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2533376,
                            "linear_total": 7077888,
                            "nnz": 2540576,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 364544,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2476032,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2840576,
                            "linear_total": 7077888,
                            "nnz": 2847872,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 237568,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 386048,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 623616,
                            "linear_total": 7077888,
                            "nnz": 629024,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 144384,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 460800,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 605184,
                            "linear_total": 7077888,
                            "nnz": 610368,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 397312,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3445760,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3843072,
                            "linear_total": 7077888,
                            "nnz": 3851072,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 666624,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3402752,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4069376,
                            "linear_total": 7077888,
                            "nnz": 4077760,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 492544,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3339264,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3831808,
                            "linear_total": 7077888,
                            "nnz": 3840000,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 519168,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3194880,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3714048,
                            "linear_total": 7077888,
                            "nnz": 3722272,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 448512,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2751488,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3200000,
                            "linear_total": 7077888,
                            "nnz": 3207744,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 576512,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1839104,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2415616,
                            "linear_total": 7077888,
                            "nnz": 2422848,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 271360,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 940032,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1211392,
                            "linear_total": 7077888,
                            "nnz": 1217440,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 317440,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 302080,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 619520,
                            "linear_total": 7077888,
                            "nnz": 624768,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 29507584,
                    "linear_sparsity": 65.25848765432099,
                    "linear_total": 84934656,
                    "nnz": 53430466,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "3": [
                            2,
                            3,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            8,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            6,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 50.93314103235074
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 85.88451743537976,
            "fill_rate": 0.34741512345679015,
            "speedup": 1.8657923656745288
        },
        {
            "annotate": "36",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.68211920529801,
                    "f1": 86.11161494070976
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_stl50_est5000_rn-output__squad_test3_dpm-sigmoied_threshold_apme-sigmoied_threshold_aowd0_bm1_dbr32_dbc32_abr32_abc32_it0_fw10_r-l1_rfl40_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-85000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 32,
                    "dense_block_rows": 32,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 40
                },
                "speed": {
                    "cuda_eval_elapsed_time": 21.577418830871583,
                    "eval_elapsed_time": 28.903804030269384
                },
                "speedup": 1.7886473497076825,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 459776,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2385920,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2845696,
                            "linear_total": 7077888,
                            "nnz": 2853376,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 374784,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2798592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3173376,
                            "linear_total": 7077888,
                            "nnz": 3180960,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 254976,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 416768,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 671744,
                            "linear_total": 7077888,
                            "nnz": 677184,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 165888,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 466944,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 632832,
                            "linear_total": 7077888,
                            "nnz": 638080,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 411648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3454976,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3866624,
                            "linear_total": 7077888,
                            "nnz": 3874720,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 727040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3496960,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4224000,
                            "linear_total": 7077888,
                            "nnz": 4232576,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 541696,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3412992,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3954688,
                            "linear_total": 7077888,
                            "nnz": 3962976,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 545792,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3447808,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3993600,
                            "linear_total": 7077888,
                            "nnz": 4001952,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 493568,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2933760,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3427328,
                            "linear_total": 7077888,
                            "nnz": 3435296,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 641024,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2000896,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2641920,
                            "linear_total": 7077888,
                            "nnz": 2649440,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 288768,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1004544,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1293312,
                            "linear_total": 7077888,
                            "nnz": 1299552,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 338944,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 339968,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 678912,
                            "linear_total": 7077888,
                            "nnz": 684288,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 31404032,
                    "linear_sparsity": 63.025655864197525,
                    "linear_total": 84934656,
                    "nnz": 55329122,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "3": [
                            2,
                            3,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            8,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            6,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 49.1895461668281
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.11161494070976,
            "fill_rate": 0.3697434413580247,
            "speedup": 1.7886473497076825
        },
        {
            "annotate": "36",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.96594134342479,
                    "f1": 86.01491496793933
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_stl50_est5000_rn-output__squad_test3_dpm-sigmoied_threshold_apme-sigmoied_threshold_aowd0_bm1_dbr32_dbc32_abr32_abc32_it0_fw10_r-l1_rfl40_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-90000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 32,
                    "dense_block_rows": 32,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 40
                },
                "speed": {
                    "cuda_eval_elapsed_time": 21.28239717102051,
                    "eval_elapsed_time": 28.641465611290187
                },
                "speedup": 1.8134420053923117,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 435200,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2226176,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2661376,
                            "linear_total": 7077888,
                            "nnz": 2668768,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 359424,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2727936,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3087360,
                            "linear_total": 7077888,
                            "nnz": 3094912,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 252928,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 411648,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 664576,
                            "linear_total": 7077888,
                            "nnz": 670048,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 158720,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 487424,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 646144,
                            "linear_total": 7077888,
                            "nnz": 651360,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 421888,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3473408,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3895296,
                            "linear_total": 7077888,
                            "nnz": 3903360,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 710656,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3451904,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4162560,
                            "linear_total": 7077888,
                            "nnz": 4171136,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 547840,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3437568,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3985408,
                            "linear_total": 7077888,
                            "nnz": 3993664,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 556032,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3325952,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3881984,
                            "linear_total": 7077888,
                            "nnz": 3890304,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 512000,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2828288,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3340288,
                            "linear_total": 7077888,
                            "nnz": 3348128,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 622592,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1991680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2614272,
                            "linear_total": 7077888,
                            "nnz": 2621728,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 276480,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 979968,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1256448,
                            "linear_total": 7077888,
                            "nnz": 1262560,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 337920,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 330752,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 668672,
                            "linear_total": 7077888,
                            "nnz": 674016,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 30864384,
                    "linear_sparsity": 63.66102430555556,
                    "linear_total": 84934656,
                    "nnz": 54788706,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "3": [
                            2,
                            3,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            8,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            6,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 49.68582699012958
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.01491496793933,
            "fill_rate": 0.3633897569444444,
            "speedup": 1.8134420053923117
        },
        {
            "annotate": "67",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 81.51371807000946,
                    "f1": 88.67903677006836
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_stl50_est5000_rn-output__squad_test3_dpm-sigmoied_threshold_apme-sigmoied_threshold_aowd0_bm1_dbr32_dbc32_abr32_abc32_it0_fw10_r-l1_rfl5_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 32,
                    "dense_block_rows": 32,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 5
                },
                "speed": {
                    "cuda_eval_elapsed_time": 31.30978426361084,
                    "eval_elapsed_time": 38.71227815328166
                },
                "speedup": 1.232662374177603,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 804864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4457472,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5262336,
                            "linear_total": 7077888,
                            "nnz": 5270944,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 771072,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4464640,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5235712,
                            "linear_total": 7077888,
                            "nnz": 5244352,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 614400,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1607680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2222080,
                            "linear_total": 7077888,
                            "nnz": 2229792,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 389120,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1230848,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1619968,
                            "linear_total": 7077888,
                            "nnz": 1626624,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1152000,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4517888,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5669888,
                            "linear_total": 7077888,
                            "nnz": 5679008,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1312768,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4562944,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5875712,
                            "linear_total": 7077888,
                            "nnz": 5885024,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1501184,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4555776,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 6056960,
                            "linear_total": 7077888,
                            "nnz": 6066560,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1377280,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4520960,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5898240,
                            "linear_total": 7077888,
                            "nnz": 5907712,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1357824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4473856,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5831680,
                            "linear_total": 7077888,
                            "nnz": 5841056,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 1192960,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4290560,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5483520,
                            "linear_total": 7077888,
                            "nnz": 5492704,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 1069056,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3889152,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4958208,
                            "linear_total": 7077888,
                            "nnz": 4967392,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 718848,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2518016,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3236864,
                            "linear_total": 7077888,
                            "nnz": 3245312,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 57351168,
                    "linear_sparsity": 32.47612847222222,
                    "linear_total": 84934656,
                    "nnz": 81295202,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            6,
                            7,
                            8,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            2
                        ],
                        "5": [
                            1,
                            2,
                            6
                        ],
                        "6": [
                            2,
                            3,
                            7
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            8,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 25.344087186502197
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.67903677006836,
            "fill_rate": 0.6752387152777778,
            "speedup": 1.232662374177603
        },
        {
            "annotate": "68",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 81.47587511825922,
                    "f1": 88.73698799207777
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_stl50_est5000_rn-output__squad_test3_dpm-sigmoied_threshold_apme-sigmoied_threshold_aowd0_bm1_dbr32_dbc32_abr32_abc32_it0_fw10_r-l1_rfl5_dl0.25_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-80000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 32,
                    "dense_block_rows": 32,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 5
                },
                "speed": {
                    "cuda_eval_elapsed_time": 31.817585739135744,
                    "eval_elapsed_time": 39.2419764213264
                },
                "speedup": 1.2129893613486789,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 921600,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4316160,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5237760,
                            "linear_total": 7077888,
                            "nnz": 5246464,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 829440,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4311040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5140480,
                            "linear_total": 7077888,
                            "nnz": 5149152,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 671744,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2001920,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2673664,
                            "linear_total": 7077888,
                            "nnz": 2681696,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 409600,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1304576,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1714176,
                            "linear_total": 7077888,
                            "nnz": 1720960,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1221632,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4419584,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5641216,
                            "linear_total": 7077888,
                            "nnz": 5650432,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1386496,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4429824,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5816320,
                            "linear_total": 7077888,
                            "nnz": 5825632,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1540096,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4457472,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5997568,
                            "linear_total": 7077888,
                            "nnz": 6007200,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1548288,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4420608,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5968896,
                            "linear_total": 7077888,
                            "nnz": 5978496,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1364992,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4320256,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5685248,
                            "linear_total": 7077888,
                            "nnz": 5694656,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 1272832,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4186112,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5458944,
                            "linear_total": 7077888,
                            "nnz": 5468160,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 1173504,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3787776,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4961280,
                            "linear_total": 7077888,
                            "nnz": 4970656,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 727040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2839552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3566592,
                            "linear_total": 7077888,
                            "nnz": 3575200,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 57862144,
                    "linear_sparsity": 31.87451774691358,
                    "linear_total": 84934656,
                    "nnz": 81807426,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            6,
                            7,
                            8,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            2
                        ],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            2,
                            3,
                            7
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 24.873695953757846
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.73698799207777,
            "fill_rate": 0.6812548225308642,
            "speedup": 1.2129893613486789
        }
    ],
    "Full block method, bs= 32x32, dl=0.5": [
        {
            "annotate": "49",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.48249763481552,
                    "f1": 87.91705961229685
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_stl50_est5000_rn-output__squad_test3_dpm-sigmoied_threshold_apme-sigmoied_threshold_aowd0_bm1_dbr32_dbc32_abr32_abc32_it0_fw10_r-l1_rfl10_dl0.5_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 32,
                    "dense_block_rows": 32,
                    "dense_lambda": 0.5,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 26.4900548248291,
                    "eval_elapsed_time": 33.8130349079147
                },
                "speedup": 1.4569389629649467,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 634880,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3140608,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3775488,
                            "linear_total": 7077888,
                            "nnz": 3783744,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 602112,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3477504,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4079616,
                            "linear_total": 7077888,
                            "nnz": 4087936,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 456704,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 494592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 951296,
                            "linear_total": 7077888,
                            "nnz": 957408,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 289792,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 541696,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 831488,
                            "linear_total": 7077888,
                            "nnz": 837280,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1008640,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3929088,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4937728,
                            "linear_total": 7077888,
                            "nnz": 4946688,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1197056,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4089856,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5286912,
                            "linear_total": 7077888,
                            "nnz": 5296064,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1181696,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3953664,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5135360,
                            "linear_total": 7077888,
                            "nnz": 5144608,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1005568,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4006912,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5012480,
                            "linear_total": 7077888,
                            "nnz": 5021440,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1043456,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3677184,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4720640,
                            "linear_total": 7077888,
                            "nnz": 4729664,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 931840,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2777088,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3708928,
                            "linear_total": 7077888,
                            "nnz": 3717120,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 862208,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1448960,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2311168,
                            "linear_total": 7077888,
                            "nnz": 2318752,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 600064,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 458752,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1058816,
                            "linear_total": 7077888,
                            "nnz": 1064960,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 41809920,
                    "linear_sparsity": 50.774016203703695,
                    "linear_total": 84934656,
                    "nnz": 65744386,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 39.6248852522324
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.91705961229685,
            "fill_rate": 0.492259837962963,
            "speedup": 1.4569389629649467
        },
        {
            "annotate": "35",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.94985808893094,
                    "f1": 86.768721062838
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_stl50_est5000_rn-output__squad_test3_dpm-sigmoied_threshold_apme-sigmoied_threshold_aowd0_bm1_dbr32_dbc32_abr32_abc32_it0_fw10_r-l1_rfl20_dl0.5_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 32,
                    "dense_block_rows": 32,
                    "dense_lambda": 0.5,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20
                },
                "speed": {
                    "cuda_eval_elapsed_time": 21.874919250488283,
                    "eval_elapsed_time": 29.121937923133373
                },
                "speedup": 1.7643216216448254,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 547840,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1844224,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2392064,
                            "linear_total": 7077888,
                            "nnz": 2399200,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 546816,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2172928,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2719744,
                            "linear_total": 7077888,
                            "nnz": 2727040,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 356352,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 392192,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 748544,
                            "linear_total": 7077888,
                            "nnz": 754336,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 217088,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 434176,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 651264,
                            "linear_total": 7077888,
                            "nnz": 656736,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 675840,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3196928,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3872768,
                            "linear_total": 7077888,
                            "nnz": 3881056,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 965632,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3111936,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4077568,
                            "linear_total": 7077888,
                            "nnz": 4086176,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 896000,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3107840,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4003840,
                            "linear_total": 7077888,
                            "nnz": 4012352,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 696320,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3136512,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3832832,
                            "linear_total": 7077888,
                            "nnz": 3841184,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 755712,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2525184,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3280896,
                            "linear_total": 7077888,
                            "nnz": 3288800,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 799744,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1711104,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2510848,
                            "linear_total": 7077888,
                            "nnz": 2518176,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 509952,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 747520,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1257472,
                            "linear_total": 7077888,
                            "nnz": 1263808,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 420864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 261120,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 681984,
                            "linear_total": 7077888,
                            "nnz": 687456,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 30029824,
                    "linear_sparsity": 64.6436149691358,
                    "linear_total": 84934656,
                    "nnz": 53955042,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 50.45140657377771
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.768721062838,
            "fill_rate": 0.353563850308642,
            "speedup": 1.7643216216448254
        },
        {
            "annotate": "22",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 76.92526017029329,
                    "f1": 85.21713644985097
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_stl50_est5000_rn-output__squad_test3_dpm-sigmoied_threshold_apme-sigmoied_threshold_aowd0_bm1_dbr32_dbc32_abr32_abc32_it0_fw10_r-l1_rfl40_dl0.5_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-105000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 32,
                    "dense_block_rows": 32,
                    "dense_lambda": 0.5,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 40
                },
                "speed": {
                    "cuda_eval_elapsed_time": 17.63341423416138,
                    "eval_elapsed_time": 24.82955563813448
                },
                "speedup": 2.1887078981336363,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 443392,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1026048,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1469440,
                            "linear_total": 7077888,
                            "nnz": 1475744,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 396288,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1296384,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1692672,
                            "linear_total": 7077888,
                            "nnz": 1699040,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 237568,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 308224,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 545792,
                            "linear_total": 7077888,
                            "nnz": 551104,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 152576,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 315392,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 467968,
                            "linear_total": 7077888,
                            "nnz": 472992,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 578560,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2113536,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2692096,
                            "linear_total": 7077888,
                            "nnz": 2699456,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 755712,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1973248,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2728960,
                            "linear_total": 7077888,
                            "nnz": 2736352,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 565248,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1966080,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2531328,
                            "linear_total": 7077888,
                            "nnz": 2538624,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 546816,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1887232,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2434048,
                            "linear_total": 7077888,
                            "nnz": 2441248,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 476160,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1502208,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1978368,
                            "linear_total": 7077888,
                            "nnz": 1985120,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 637952,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1000448,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1638400,
                            "linear_total": 7077888,
                            "nnz": 1644928,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 310272,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 310272,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 620544,
                            "linear_total": 7077888,
                            "nnz": 626048,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 313344,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 144384,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 457728,
                            "linear_total": 7077888,
                            "nnz": 462720,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 19257344,
                    "linear_sparsity": 77.3268711419753,
                    "linear_total": 84934656,
                    "nnz": 43172098,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            3,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            6,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 60.35371946964616
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 85.21713644985097,
            "fill_rate": 0.22673128858024694,
            "speedup": 2.1887078981336363
        },
        {
            "annotate": "22",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.08609271523179,
                    "f1": 85.20287591064626
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_stl50_est5000_rn-output__squad_test3_dpm-sigmoied_threshold_apme-sigmoied_threshold_aowd0_bm1_dbr32_dbc32_abr32_abc32_it0_fw10_r-l1_rfl40_dl0.5_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 32,
                    "dense_block_rows": 32,
                    "dense_lambda": 0.5,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 40
                },
                "speed": {
                    "cuda_eval_elapsed_time": 17.564620765686037,
                    "eval_elapsed_time": 24.740368818864226
                },
                "speedup": 2.1972801758844964,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 455680,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1007616,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1463296,
                            "linear_total": 7077888,
                            "nnz": 1469568,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 399360,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1300480,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1699840,
                            "linear_total": 7077888,
                            "nnz": 1706144,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 240640,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 305152,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 545792,
                            "linear_total": 7077888,
                            "nnz": 551104,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 144384,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 329728,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 474112,
                            "linear_total": 7077888,
                            "nnz": 479136,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 544768,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2180096,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2724864,
                            "linear_total": 7077888,
                            "nnz": 2732224,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 731136,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1939456,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2670592,
                            "linear_total": 7077888,
                            "nnz": 2677952,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 557056,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1941504,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2498560,
                            "linear_total": 7077888,
                            "nnz": 2505856,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 527360,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1880064,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2407424,
                            "linear_total": 7077888,
                            "nnz": 2414624,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 472064,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1456128,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1928192,
                            "linear_total": 7077888,
                            "nnz": 1934880,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 607232,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 977920,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1585152,
                            "linear_total": 7077888,
                            "nnz": 1591680,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 289792,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 317440,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 607232,
                            "linear_total": 7077888,
                            "nnz": 612736,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 308224,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 147456,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 455680,
                            "linear_total": 7077888,
                            "nnz": 460704,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 19060736,
                    "linear_sparsity": 77.55835262345678,
                    "linear_total": 84934656,
                    "nnz": 42975330,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            3,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            6,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 60.53441764482857
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 85.20287591064626,
            "fill_rate": 0.22441647376543217,
            "speedup": 2.1972801758844964
        },
        {
            "annotate": "25",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.01986754966887,
                    "f1": 85.2617013700351
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_stl50_est5000_rn-output__squad_test3_dpm-sigmoied_threshold_apme-sigmoied_threshold_aowd0_bm1_dbr32_dbc32_abr32_abc32_it0_fw10_r-l1_rfl40_dl0.5_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-80000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 32,
                    "dense_block_rows": 32,
                    "dense_lambda": 0.5,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 40
                },
                "speed": {
                    "cuda_eval_elapsed_time": 18.277880432128907,
                    "eval_elapsed_time": 25.53750513214618
                },
                "speedup": 2.1115354785629177,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 480256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1234944,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1715200,
                            "linear_total": 7077888,
                            "nnz": 1721696,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 400384,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1495040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1895424,
                            "linear_total": 7077888,
                            "nnz": 1901888,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 267264,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 326656,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 593920,
                            "linear_total": 7077888,
                            "nnz": 599328,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 163840,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 337920,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 501760,
                            "linear_total": 7077888,
                            "nnz": 506880,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 594944,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2417664,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3012608,
                            "linear_total": 7077888,
                            "nnz": 3020160,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 813056,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2281472,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3094528,
                            "linear_total": 7077888,
                            "nnz": 3102368,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 599040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2163712,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2762752,
                            "linear_total": 7077888,
                            "nnz": 2770208,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 562176,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2145280,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2707456,
                            "linear_total": 7077888,
                            "nnz": 2714848,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 531456,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1701888,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2233344,
                            "linear_total": 7077888,
                            "nnz": 2240288,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 678912,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1062912,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1741824,
                            "linear_total": 7077888,
                            "nnz": 1748512,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 338944,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 370688,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 709632,
                            "linear_total": 7077888,
                            "nnz": 715264,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 359424,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 164864,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 524288,
                            "linear_total": 7077888,
                            "nnz": 529504,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 21492736,
                    "linear_sparsity": 74.6949749228395,
                    "linear_total": 84934656,
                    "nnz": 45409666,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            3,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            6,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 58.29889117212532
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 85.2617013700351,
            "fill_rate": 0.2530502507716049,
            "speedup": 2.1115354785629177
        },
        {
            "annotate": "24",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 76.98202459791864,
                    "f1": 85.22056943761015
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_stl50_est5000_rn-output__squad_test3_dpm-sigmoied_threshold_apme-sigmoied_threshold_aowd0_bm1_dbr32_dbc32_abr32_abc32_it0_fw10_r-l1_rfl40_dl0.5_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-90000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 32,
                    "dense_block_rows": 32,
                    "dense_lambda": 0.5,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 40
                },
                "speed": {
                    "cuda_eval_elapsed_time": 17.92396342086792,
                    "eval_elapsed_time": 25.119796799961478
                },
                "speedup": 2.153228730674472,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 458752,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1139712,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1598464,
                            "linear_total": 7077888,
                            "nnz": 1604832,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 398336,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1427456,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1825792,
                            "linear_total": 7077888,
                            "nnz": 1832224,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 271360,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 326656,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 598016,
                            "linear_total": 7077888,
                            "nnz": 603232,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 162816,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 347136,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 509952,
                            "linear_total": 7077888,
                            "nnz": 515008,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 596992,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2257920,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2854912,
                            "linear_total": 7077888,
                            "nnz": 2862432,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 781312,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2123776,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2905088,
                            "linear_total": 7077888,
                            "nnz": 2912704,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 620544,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2023424,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2643968,
                            "linear_total": 7077888,
                            "nnz": 2651296,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 573440,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1970176,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2543616,
                            "linear_total": 7077888,
                            "nnz": 2550880,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 460800,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1588224,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2049024,
                            "linear_total": 7077888,
                            "nnz": 2055904,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 638976,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1069056,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1708032,
                            "linear_total": 7077888,
                            "nnz": 1714656,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 307200,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 359424,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 666624,
                            "linear_total": 7077888,
                            "nnz": 672160,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 327680,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 161792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 489472,
                            "linear_total": 7077888,
                            "nnz": 494624,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 20392960,
                    "linear_sparsity": 75.98982445987654,
                    "linear_total": 84934656,
                    "nnz": 44308674,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            3,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            6,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 59.309966373837206
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 85.22056943761015,
            "fill_rate": 0.24010175540123457,
            "speedup": 2.153228730674472
        }
    ],
    "Full block method, bs= 32x32, dl=1.0": [
        {
            "annotate": "43",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.64049195837275,
                    "f1": 87.40026291426761
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test/hp_es-steps_nte20_ls250_est5000_dpm-sigmoied_threshold_apme-sigmoied_threshold_dbr32_dbc32_abr32_abc32_it0_fw10_r-l1_rfl10_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 32,
                    "dense_block_rows": 32,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 24.480328201293947,
                    "eval_elapsed_time": 31.782106802333146
                },
                "speedup": 1.5765472050873537,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 614400,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2607104,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3221504,
                            "linear_total": 7077888,
                            "nnz": 3231488,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 604160,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2899968,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3504128,
                            "linear_total": 7077888,
                            "nnz": 3514112,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 401408,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 451584,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 852992,
                            "linear_total": 7077888,
                            "nnz": 862976,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 244736,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 501760,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 746496,
                            "linear_total": 7077888,
                            "nnz": 756480,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 730112,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3708928,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4439040,
                            "linear_total": 7077888,
                            "nnz": 4449024,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1044480,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3815424,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4859904,
                            "linear_total": 7077888,
                            "nnz": 4869888,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1012736,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3722240,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4734976,
                            "linear_total": 7077888,
                            "nnz": 4744960,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 882688,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3777536,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4660224,
                            "linear_total": 7077888,
                            "nnz": 4670208,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 980992,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3258368,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4239360,
                            "linear_total": 7077888,
                            "nnz": 4249344,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 903168,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2234368,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3137536,
                            "linear_total": 7077888,
                            "nnz": 3147520,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 710656,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1124352,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1835008,
                            "linear_total": 7077888,
                            "nnz": 1844992,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 552960,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 324608,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 877568,
                            "linear_total": 7077888,
                            "nnz": 887552,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 37108736,
                    "linear_sparsity": 56.309076003086425,
                    "linear_total": 84934656,
                    "nnz": 61067266,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            1,
                            2,
                            11,
                            6
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            3,
                            4,
                            5,
                            8
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 43.920030037508496
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "output_dir": "output/squad_test",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 79.84862819299906,
                    "f1": 87.52317853046331
                }
            },
            "f1": 87.40026291426761,
            "fill_rate": 0.4369092399691358,
            "speedup": 1.5765472050873537
        },
        {
            "annotate": "37",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.82024597918638,
                    "f1": 87.30735739624531
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_stl50_est5000_rn-output__squad_test3_dpm-sigmoied_threshold_apme-sigmoied_threshold_aowd0_bm1_dbr32_dbc32_abr32_abc32_it0_fw10_r-l1_rfl10_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 32,
                    "dense_block_rows": 32,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 24.124949531555178,
                    "eval_elapsed_time": 31.406295038294047
                },
                "speedup": 1.599770932365684,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 889856,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1492992,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2382848,
                            "linear_total": 7077888,
                            "nnz": 2390080,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 717824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1850368,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2568192,
                            "linear_total": 7077888,
                            "nnz": 2575360,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 489472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 328704,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 818176,
                            "linear_total": 7077888,
                            "nnz": 824096,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 331776,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 388096,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 719872,
                            "linear_total": 7077888,
                            "nnz": 725696,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1113088,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2802688,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3915776,
                            "linear_total": 7077888,
                            "nnz": 3924128,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1297408,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2961408,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4258816,
                            "linear_total": 7077888,
                            "nnz": 4267520,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1402880,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2897920,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4300800,
                            "linear_total": 7077888,
                            "nnz": 4309792,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1157120,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2873344,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4030464,
                            "linear_total": 7077888,
                            "nnz": 4039136,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1187840,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2473984,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3661824,
                            "linear_total": 7077888,
                            "nnz": 3670208,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 979968,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1527808,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2507776,
                            "linear_total": 7077888,
                            "nnz": 2514976,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 952320,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 610304,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1562624,
                            "linear_total": 7077888,
                            "nnz": 1569312,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 642048,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 223232,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 865280,
                            "linear_total": 7077888,
                            "nnz": 871008,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 31592448,
                    "linear_sparsity": 62.80381944444444,
                    "linear_total": 84934656,
                    "nnz": 55520034,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            7,
                            8,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            1,
                            2
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7
                        ],
                        "6": [
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 49.0142257386059
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.30735739624531,
            "fill_rate": 0.3719618055555556,
            "speedup": 1.599770932365684
        },
        {
            "annotate": "40",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.64995269631031,
                    "f1": 87.30139925832849
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_stl50_est5000_rn-output__squad_test3_dpm-sigmoied_threshold_apme-sigmoied_threshold_aowd0_bm1_dbr32_dbc32_abr32_abc32_it0_fw10_r-l1_rfl10_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-80000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 32,
                    "dense_block_rows": 32,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 25.313612380981446,
                    "eval_elapsed_time": 32.61186430603266
                },
                "speedup": 1.524649758576841,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 934912,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1777664,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2712576,
                            "linear_total": 7077888,
                            "nnz": 2720032,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 738304,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2168832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2907136,
                            "linear_total": 7077888,
                            "nnz": 2914688,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 530432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 343040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 873472,
                            "linear_total": 7077888,
                            "nnz": 879520,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 378880,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 421888,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 800768,
                            "linear_total": 7077888,
                            "nnz": 806656,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1162240,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3039232,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4201472,
                            "linear_total": 7077888,
                            "nnz": 4210176,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1366016,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3165184,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4531200,
                            "linear_total": 7077888,
                            "nnz": 4540192,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1484800,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3182592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4667392,
                            "linear_total": 7077888,
                            "nnz": 4676672,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1414144,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3185664,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4599808,
                            "linear_total": 7077888,
                            "nnz": 4609152,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1256448,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2572288,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3828736,
                            "linear_total": 7077888,
                            "nnz": 3837312,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 991232,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1668096,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2659328,
                            "linear_total": 7077888,
                            "nnz": 2666688,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 966656,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 688128,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1654784,
                            "linear_total": 7077888,
                            "nnz": 1661632,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 691200,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 236544,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 927744,
                            "linear_total": 7077888,
                            "nnz": 933568,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 34364416,
                    "linear_sparsity": 59.540171682098766,
                    "linear_total": 84934656,
                    "nnz": 58295010,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            7,
                            8,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            1,
                            2
                        ],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 46.46587895775224
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 79.82024597918638,
                    "f1": 87.41794090203474
                }
            },
            "f1": 87.30139925832849,
            "fill_rate": 0.40459828317901236,
            "speedup": 1.524649758576841
        },
        {
            "annotate": "38",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.90539262062441,
                    "f1": 87.36378709007766
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_stl50_est5000_rn-output__squad_test3_dpm-sigmoied_threshold_apme-sigmoied_threshold_aowd0_bm1_dbr32_dbc32_abr32_abc32_it0_fw10_r-l1_rfl10_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-90000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 32,
                    "dense_block_rows": 32,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 24.748493873596193,
                    "eval_elapsed_time": 32.03074289299548
                },
                "speedup": 1.559464313363606,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 949248,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1635328,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2584576,
                            "linear_total": 7077888,
                            "nnz": 2591936,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 750592,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2048000,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2798592,
                            "linear_total": 7077888,
                            "nnz": 2805952,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 509952,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 352256,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 862208,
                            "linear_total": 7077888,
                            "nnz": 868096,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 363520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 420864,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 784384,
                            "linear_total": 7077888,
                            "nnz": 790208,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1123328,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2895872,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4019200,
                            "linear_total": 7077888,
                            "nnz": 4027808,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1306624,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2967552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4274176,
                            "linear_total": 7077888,
                            "nnz": 4282976,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1475584,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3105792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4581376,
                            "linear_total": 7077888,
                            "nnz": 4590592,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1285120,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2934784,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4219904,
                            "linear_total": 7077888,
                            "nnz": 4229024,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1235968,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2500608,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3736576,
                            "linear_total": 7077888,
                            "nnz": 3745056,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1604608,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2587648,
                            "linear_total": 7077888,
                            "nnz": 2594944,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 965632,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 661504,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1627136,
                            "linear_total": 7077888,
                            "nnz": 1633888,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 650240,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 230400,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 880640,
                            "linear_total": 7077888,
                            "nnz": 886432,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 32956416,
                    "linear_sparsity": 61.19791666666667,
                    "linear_total": 84934656,
                    "nnz": 56885634,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            7,
                            8,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            1,
                            2
                        ],
                        "5": [
                            1,
                            2,
                            6
                        ],
                        "6": [
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 47.76015277944021
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.36378709007766,
            "fill_rate": 0.38802083333333326,
            "speedup": 1.559464313363606
        },
        {
            "annotate": "24",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.8713339640492,
                    "f1": 85.84893170709621
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_stl50_est5000_rn-output__squad_test3_dpm-sigmoied_threshold_apme-sigmoied_threshold_aowd0_bm1_dbr32_dbc32_abr32_abc32_it0_fw10_r-l1_rfl20_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 32,
                    "dense_block_rows": 32,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20
                },
                "speed": {
                    "cuda_eval_elapsed_time": 19.24458102798462,
                    "eval_elapsed_time": 26.45731420116499
                },
                "speedup": 2.0054680821187447,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 647168,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 789504,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1436672,
                            "linear_total": 7077888,
                            "nnz": 1443040,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 591872,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1206272,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1798144,
                            "linear_total": 7077888,
                            "nnz": 1804608,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 359424,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 263168,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 622592,
                            "linear_total": 7077888,
                            "nnz": 628000,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 240640,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 271360,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 512000,
                            "linear_total": 7077888,
                            "nnz": 517312,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 843776,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1739776,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2583552,
                            "linear_total": 7077888,
                            "nnz": 2590848,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1118208,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1857536,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2975744,
                            "linear_total": 7077888,
                            "nnz": 2983488,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 913408,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1760256,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2673664,
                            "linear_total": 7077888,
                            "nnz": 2681056,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 791552,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1718272,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2509824,
                            "linear_total": 7077888,
                            "nnz": 2517088,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 755712,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1330176,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2085888,
                            "linear_total": 7077888,
                            "nnz": 2092864,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 827392,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 904192,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1731584,
                            "linear_total": 7077888,
                            "nnz": 1738144,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 726016,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 257024,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 983040,
                            "linear_total": 7077888,
                            "nnz": 989184,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 464896,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 118784,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 583680,
                            "linear_total": 7077888,
                            "nnz": 588928,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 20496384,
                    "linear_sparsity": 75.86805555555556,
                    "linear_total": 84934656,
                    "nnz": 44413282,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            2,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 59.21390159343854
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 85.84893170709621,
            "fill_rate": 0.24131944444444442,
            "speedup": 2.0054680821187447
        },
        {
            "annotate": "25",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.92809839167455,
                    "f1": 85.97854187426412
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_stl50_est5000_rn-output__squad_test3_dpm-sigmoied_threshold_apme-sigmoied_threshold_aowd0_bm1_dbr32_dbc32_abr32_abc32_it0_fw10_r-l1_rfl20_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-90000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 32,
                    "dense_block_rows": 32,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20
                },
                "speed": {
                    "cuda_eval_elapsed_time": 19.635457836151122,
                    "eval_elapsed_time": 26.92565976222977
                },
                "speedup": 1.9655458674518098,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 679936,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 869376,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1549312,
                            "linear_total": 7077888,
                            "nnz": 1555808,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 599040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1269760,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1868800,
                            "linear_total": 7077888,
                            "nnz": 1875328,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 379904,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 282624,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 662528,
                            "linear_total": 7077888,
                            "nnz": 668000,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 258048,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 290816,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 548864,
                            "linear_total": 7077888,
                            "nnz": 554208,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 875520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1863680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2739200,
                            "linear_total": 7077888,
                            "nnz": 2746656,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1137664,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1950720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3088384,
                            "linear_total": 7077888,
                            "nnz": 3096256,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1033216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1787904,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2821120,
                            "linear_total": 7077888,
                            "nnz": 2828864,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 850944,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1858560,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2709504,
                            "linear_total": 7077888,
                            "nnz": 2717024,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 798720,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1426432,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2225152,
                            "linear_total": 7077888,
                            "nnz": 2232160,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 878592,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 987136,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1865728,
                            "linear_total": 7077888,
                            "nnz": 1872352,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 782336,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 267264,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1049600,
                            "linear_total": 7077888,
                            "nnz": 1055808,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 504832,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 144384,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 649216,
                            "linear_total": 7077888,
                            "nnz": 654528,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 21777408,
                    "linear_sparsity": 74.35980902777779,
                    "linear_total": 84934656,
                    "nnz": 45695714,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            2,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 58.036204395746125
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 85.97854187426412,
            "fill_rate": 0.2564019097222221,
            "speedup": 1.9655458674518098
        },
        {
            "annotate": "18",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 76.79280983916746,
                    "f1": 85.3167029862563
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_stl50_est5000_rn-output__squad_test3_dpm-sigmoied_threshold_apme-sigmoied_threshold_aowd0_bm1_dbr32_dbc32_abr32_abc32_it0_fw10_r-l1_rfl30_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-105000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 32,
                    "dense_block_rows": 32,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 30
                },
                "speed": {
                    "cuda_eval_elapsed_time": 16.872496753692626,
                    "eval_elapsed_time": 24.01387820020318
                },
                "speedup": 2.2874144573134694,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 512000,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 512000,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1024000,
                            "linear_total": 7077888,
                            "nnz": 1029984,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 551936,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 685056,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1236992,
                            "linear_total": 7077888,
                            "nnz": 1242912,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 304128,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 197632,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 501760,
                            "linear_total": 7077888,
                            "nnz": 506976,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 197632,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 220160,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 417792,
                            "linear_total": 7077888,
                            "nnz": 422880,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 722944,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1211392,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1934336,
                            "linear_total": 7077888,
                            "nnz": 1940960,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 954368,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1397760,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2352128,
                            "linear_total": 7077888,
                            "nnz": 2359232,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 790528,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1238016,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2028544,
                            "linear_total": 7077888,
                            "nnz": 2035424,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 584704,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1295360,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1880064,
                            "linear_total": 7077888,
                            "nnz": 1886784,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 608256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1018880,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1627136,
                            "linear_total": 7077888,
                            "nnz": 1633600,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 740352,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 576512,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1316864,
                            "linear_total": 7077888,
                            "nnz": 1323104,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 510976,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 162816,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 673792,
                            "linear_total": 7077888,
                            "nnz": 679488,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 357376,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 94208,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 451584,
                            "linear_total": 7077888,
                            "nnz": 456544,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 15444992,
                    "linear_sparsity": 81.81544174382715,
                    "linear_total": 84934656,
                    "nnz": 39356610,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 63.85760078688487
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 85.3167029862563,
            "fill_rate": 0.18184558256172845,
            "speedup": 2.2874144573134694
        },
        {
            "annotate": "17",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.04824976348155,
                    "f1": 85.17930403802184
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_stl50_est5000_rn-output__squad_test3_dpm-sigmoied_threshold_apme-sigmoied_threshold_aowd0_bm1_dbr32_dbc32_abr32_abc32_it0_fw10_r-l1_rfl30_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 32,
                    "dense_block_rows": 32,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 30
                },
                "speed": {
                    "cuda_eval_elapsed_time": 16.85802384185791,
                    "eval_elapsed_time": 24.0219326200895
                },
                "speedup": 2.289378243109522,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 513024,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 519168,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1032192,
                            "linear_total": 7077888,
                            "nnz": 1038176,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 523264,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 692224,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1215488,
                            "linear_total": 7077888,
                            "nnz": 1221408,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 312320,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 206848,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 519168,
                            "linear_total": 7077888,
                            "nnz": 524256,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 186368,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 215040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 401408,
                            "linear_total": 7077888,
                            "nnz": 406528,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 683008,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1239040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1922048,
                            "linear_total": 7077888,
                            "nnz": 1928672,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 945152,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1374208,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2319360,
                            "linear_total": 7077888,
                            "nnz": 2326464,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 809984,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1235968,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2045952,
                            "linear_total": 7077888,
                            "nnz": 2052832,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 581632,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1265664,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1847296,
                            "linear_total": 7077888,
                            "nnz": 1854016,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 600064,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1007616,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1607680,
                            "linear_total": 7077888,
                            "nnz": 1614176,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 708608,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 578560,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1287168,
                            "linear_total": 7077888,
                            "nnz": 1293408,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 473088,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 158720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 631808,
                            "linear_total": 7077888,
                            "nnz": 637472,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 352256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 90112,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 442368,
                            "linear_total": 7077888,
                            "nnz": 447232,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 15271936,
                    "linear_sparsity": 82.0191936728395,
                    "linear_total": 84934656,
                    "nnz": 39183362,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 64.01669981444019
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 85.17930403802184,
            "fill_rate": 0.17980806327160492,
            "speedup": 2.289378243109522
        }
    ],
    "Full block method, bs= 4x4, dl=0.25": [
        {
            "annotate": "57",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 81.47587511825922,
                    "f1": 88.58172107792693
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v9-a4-l10--2021-01-20--19-01-04/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 4,
                    "attention_block_rows": 4,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 4,
                    "dense_block_rows": 4,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 34.2993692779541,
                    "eval_elapsed_time": 41.87211530236527
                },
                "speedup": 1.1252216532791355,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 428592,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3980096,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4408688,
                            "linear_total": 7077888,
                            "nnz": 4417456,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 545744,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4015584,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4561328,
                            "linear_total": 7077888,
                            "nnz": 4570120,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 329968,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2092032,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2422000,
                            "linear_total": 7077888,
                            "nnz": 2430724,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 190816,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1335104,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1525920,
                            "linear_total": 7077888,
                            "nnz": 1534052,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 729664,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4061440,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4791104,
                            "linear_total": 7077888,
                            "nnz": 4800292,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 851472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4062640,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4914112,
                            "linear_total": 7077888,
                            "nnz": 4923388,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 960992,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4047744,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5008736,
                            "linear_total": 7077888,
                            "nnz": 5018448,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 902768,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4006096,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4908864,
                            "linear_total": 7077888,
                            "nnz": 4918468,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 861120,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3920672,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4781792,
                            "linear_total": 7077888,
                            "nnz": 4791412,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 759664,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3732848,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4492512,
                            "linear_total": 7077888,
                            "nnz": 4501704,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 670096,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3391392,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4061488,
                            "linear_total": 7077888,
                            "nnz": 4070864,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 444064,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2661776,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3105840,
                            "linear_total": 7077888,
                            "nnz": 3114612,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 48982384,
                    "linear_sparsity": 42.329331386236504,
                    "linear_total": 84934656,
                    "nnz": 72930262,
                    "pruned_heads": {
                        "0": [
                            9
                        ],
                        "1": [
                            0,
                            8,
                            2
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            5,
                            7,
                            8,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [],
                        "5": [
                            1
                        ],
                        "6": [
                            3
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 33.025871793300276
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.58172107792693,
            "fill_rate": 0.576706686137635,
            "speedup": 1.1252216532791355
        },
        {
            "annotate": "46",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.52980132450331,
                    "f1": 88.02284574429551
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v9-a4-l20--2021-01-20--19-01-34/checkpoint-105000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 4,
                    "attention_block_rows": 4,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 4,
                    "dense_block_rows": 4,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 32.3459995803833,
                    "eval_elapsed_time": 40.03914254019037
                },
                "speedup": 1.1931736074335828,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 258016,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3584960,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3842976,
                            "linear_total": 7077888,
                            "nnz": 3851352,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 404784,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3659360,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4064144,
                            "linear_total": 7077888,
                            "nnz": 4072688,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 209136,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1083920,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1293056,
                            "linear_total": 7077888,
                            "nnz": 1301528,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 120976,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 697408,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 818384,
                            "linear_total": 7077888,
                            "nnz": 825580,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 460752,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3741328,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4202080,
                            "linear_total": 7077888,
                            "nnz": 4211032,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 577184,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3724032,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4301216,
                            "linear_total": 7077888,
                            "nnz": 4310276,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 587792,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3689648,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4277440,
                            "linear_total": 7077888,
                            "nnz": 4286684,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 530480,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3641984,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4172464,
                            "linear_total": 7077888,
                            "nnz": 4181480,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 508336,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3491408,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3999744,
                            "linear_total": 7077888,
                            "nnz": 4008976,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 486304,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3187056,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3673360,
                            "linear_total": 7077888,
                            "nnz": 3682292,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 374032,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2669344,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3043376,
                            "linear_total": 7077888,
                            "nnz": 3052288,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 276992,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1586976,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1863968,
                            "linear_total": 7077888,
                            "nnz": 1872484,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 39552208,
                    "linear_sparsity": 53.432191448447156,
                    "linear_total": 84934656,
                    "nnz": 63495382,
                    "pruned_heads": {
                        "0": [
                            9,
                            2,
                            4,
                            5
                        ],
                        "1": [
                            0,
                            8,
                            2,
                            6
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            6
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7
                        ],
                        "6": [
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            4
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 41.69021558428826
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.02284574429551,
            "fill_rate": 0.4656780855155285,
            "speedup": 1.1931736074335828
        },
        {
            "annotate": "46",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.09460737937559,
                    "f1": 87.80889686617203
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v9-a4-l20--2021-01-20--19-01-34/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 4,
                    "attention_block_rows": 4,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 4,
                    "dense_block_rows": 4,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 32.19205239105224,
                    "eval_elapsed_time": 39.82947535999119
                },
                "speedup": 1.1988795413397866,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 258016,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3584960,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3842976,
                            "linear_total": 7077888,
                            "nnz": 3851352,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 404784,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3659360,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4064144,
                            "linear_total": 7077888,
                            "nnz": 4072688,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 209136,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1083920,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1293056,
                            "linear_total": 7077888,
                            "nnz": 1301528,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 120976,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 697408,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 818384,
                            "linear_total": 7077888,
                            "nnz": 825580,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 460752,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3741328,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4202080,
                            "linear_total": 7077888,
                            "nnz": 4211032,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 577184,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3724032,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4301216,
                            "linear_total": 7077888,
                            "nnz": 4310276,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 587792,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3689648,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4277440,
                            "linear_total": 7077888,
                            "nnz": 4286684,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 530480,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3641984,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4172464,
                            "linear_total": 7077888,
                            "nnz": 4181480,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 508336,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3491408,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3999744,
                            "linear_total": 7077888,
                            "nnz": 4008976,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 486304,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3187056,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3673360,
                            "linear_total": 7077888,
                            "nnz": 3682292,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 374032,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2669344,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3043376,
                            "linear_total": 7077888,
                            "nnz": 3052288,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 276992,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1586976,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1863968,
                            "linear_total": 7077888,
                            "nnz": 1872484,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 39552208,
                    "linear_sparsity": 53.432191448447156,
                    "linear_total": 84934656,
                    "nnz": 63495382,
                    "pruned_heads": {
                        "0": [
                            9,
                            2,
                            4,
                            5
                        ],
                        "1": [
                            0,
                            8,
                            2,
                            6
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            6
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7
                        ],
                        "6": [
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            4
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 41.69021558428826
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.80889686617203,
            "fill_rate": 0.4656780855155285,
            "speedup": 1.1988795413397866
        },
        {
            "annotate": "35",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.64049195837275,
                    "f1": 87.31499809166372
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v9-a4-l40--2021-01-20--19-02-03/checkpoint-105000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 4,
                    "attention_block_rows": 4,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 4,
                    "dense_block_rows": 4,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 40.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 28.556625274658202,
                    "eval_elapsed_time": 36.13367621740326
                },
                "speedup": 1.3515039902008532,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 169136,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2961360,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3130496,
                            "linear_total": 7077888,
                            "nnz": 3138744,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 304464,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3089024,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3393488,
                            "linear_total": 7077888,
                            "nnz": 3401900,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 137920,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 522400,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 660320,
                            "linear_total": 7077888,
                            "nnz": 667764,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 82480,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 374544,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 457024,
                            "linear_total": 7077888,
                            "nnz": 463412,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 279216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3191664,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3470880,
                            "linear_total": 7077888,
                            "nnz": 3479332,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 429728,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3150736,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3580464,
                            "linear_total": 7077888,
                            "nnz": 3589288,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 314688,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3076048,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3390736,
                            "linear_total": 7077888,
                            "nnz": 3399308,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 326416,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3008016,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3334432,
                            "linear_total": 7077888,
                            "nnz": 3343056,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 281984,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2766480,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3048464,
                            "linear_total": 7077888,
                            "nnz": 3057152,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 320352,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2338640,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2658992,
                            "linear_total": 7077888,
                            "nnz": 2667712,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 200608,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1736048,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1936656,
                            "linear_total": 7077888,
                            "nnz": 1945012,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 185008,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 707152,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 892160,
                            "linear_total": 7077888,
                            "nnz": 900284,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 29954112,
                    "linear_sparsity": 64.7327564380787,
                    "linear_total": 84934656,
                    "nnz": 53891686,
                    "pruned_heads": {
                        "0": [
                            9,
                            2,
                            4,
                            5
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            2,
                            11,
                            6
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 50.50958835936713
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.31499809166372,
            "fill_rate": 0.352672435619213,
            "speedup": 1.3515039902008532
        },
        {
            "annotate": "35",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.06338694418164,
                    "f1": 86.86293366416082
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v9-a4-l40--2021-01-20--19-02-03/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 4,
                    "attention_block_rows": 4,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 4,
                    "dense_block_rows": 4,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 40.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 28.558930541992186,
                    "eval_elapsed_time": 36.152482252102345
                },
                "speedup": 1.3513948972501988,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 169136,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2961360,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3130496,
                            "linear_total": 7077888,
                            "nnz": 3138744,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 304464,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3089024,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3393488,
                            "linear_total": 7077888,
                            "nnz": 3401900,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 137920,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 522400,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 660320,
                            "linear_total": 7077888,
                            "nnz": 667764,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 82480,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 374544,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 457024,
                            "linear_total": 7077888,
                            "nnz": 463412,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 279216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3191664,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3470880,
                            "linear_total": 7077888,
                            "nnz": 3479332,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 429728,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3150736,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3580464,
                            "linear_total": 7077888,
                            "nnz": 3589288,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 314688,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3076048,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3390736,
                            "linear_total": 7077888,
                            "nnz": 3399308,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 326416,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3008016,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3334432,
                            "linear_total": 7077888,
                            "nnz": 3343056,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 281984,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2766480,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3048464,
                            "linear_total": 7077888,
                            "nnz": 3057152,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 320352,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2338640,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2658992,
                            "linear_total": 7077888,
                            "nnz": 2667712,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 200608,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1736048,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1936656,
                            "linear_total": 7077888,
                            "nnz": 1945012,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 185008,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 707152,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 892160,
                            "linear_total": 7077888,
                            "nnz": 900284,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 29954112,
                    "linear_sparsity": 64.7327564380787,
                    "linear_total": 84934656,
                    "nnz": 53891686,
                    "pruned_heads": {
                        "0": [
                            9,
                            2,
                            4,
                            5
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            2,
                            11,
                            6
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 50.50958835936713
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 79.13907284768212,
                    "f1": 86.97173787941202
                }
            },
            "f1": 86.86293366416082,
            "fill_rate": 0.352672435619213,
            "speedup": 1.3513948972501988
        }
    ],
    "Full block method, bs= 4x4, dl=1.0": [
        {
            "annotate": "35",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.9271523178808,
                    "f1": 88.21768668110452
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v11-a4-l10-dl1--2021-01-24--15-47-42/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 4,
                    "attention_block_rows": 4,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 4,
                    "dense_block_rows": 4,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 33.44704815673828,
                    "eval_elapsed_time": 40.718972705770284
                },
                "speedup": 1.1538953400165994,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 528912,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2409360,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2938272,
                            "linear_total": 7077888,
                            "nnz": 2947176,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 618448,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2534112,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3152560,
                            "linear_total": 7077888,
                            "nnz": 3161488,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 357616,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 309216,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 666832,
                            "linear_total": 7077888,
                            "nnz": 674224,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 219536,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 276672,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 496208,
                            "linear_total": 7077888,
                            "nnz": 502932,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 835904,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2670704,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3506608,
                            "linear_total": 7077888,
                            "nnz": 3515852,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 958400,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2670800,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3629200,
                            "linear_total": 7077888,
                            "nnz": 3638572,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1091248,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2620432,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3711680,
                            "linear_total": 7077888,
                            "nnz": 3721476,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1029984,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2535968,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3565952,
                            "linear_total": 7077888,
                            "nnz": 3575596,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 964544,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2286960,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3251504,
                            "linear_total": 7077888,
                            "nnz": 3261196,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 813552,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1727488,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2541040,
                            "linear_total": 7077888,
                            "nnz": 2550228,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 744336,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1096768,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1841104,
                            "linear_total": 7077888,
                            "nnz": 1850448,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 473664,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 386800,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 860464,
                            "linear_total": 7077888,
                            "nnz": 868284,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 30161424,
                    "linear_sparsity": 64.48867232711225,
                    "linear_total": 84934656,
                    "nnz": 54106194,
                    "pruned_heads": {
                        "0": [],
                        "1": [
                            0,
                            2
                        ],
                        "10": [
                            1,
                            4,
                            5
                        ],
                        "11": [
                            8,
                            11,
                            5,
                            7
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [],
                        "5": [
                            1
                        ],
                        "6": [
                            3
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 50.31259899035372
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.21768668110452,
            "fill_rate": 0.3551132767288775,
            "speedup": 1.1538953400165994
        },
        {
            "annotate": "22",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.72563859981078,
                    "f1": 87.37325813950282
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v11-a4-l20-dl1--2021-01-24--15-48-09/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 4,
                    "attention_block_rows": 4,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 4,
                    "dense_block_rows": 4,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 29.62903995513916,
                    "eval_elapsed_time": 37.18844554480165
                },
                "speedup": 1.302586687378539,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 369024,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1492400,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1861424,
                            "linear_total": 7077888,
                            "nnz": 1870032,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 467072,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1640528,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2107600,
                            "linear_total": 7077888,
                            "nnz": 2116180,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 242352,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 173264,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 415616,
                            "linear_total": 7077888,
                            "nnz": 422044,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 157280,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 168800,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 326080,
                            "linear_total": 7077888,
                            "nnz": 332188,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 643248,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1776032,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2419280,
                            "linear_total": 7077888,
                            "nnz": 2428408,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 744560,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1758000,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2502560,
                            "linear_total": 7077888,
                            "nnz": 2511776,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 773760,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1672784,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2446544,
                            "linear_total": 7077888,
                            "nnz": 2456100,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 636208,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1581568,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2217776,
                            "linear_total": 7077888,
                            "nnz": 2226980,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 605664,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1321040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1926704,
                            "linear_total": 7077888,
                            "nnz": 1935868,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 548160,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 906384,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1454544,
                            "linear_total": 7077888,
                            "nnz": 1463048,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 486464,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 524352,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1010816,
                            "linear_total": 7077888,
                            "nnz": 1019180,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 306864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 180544,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 487408,
                            "linear_total": 7077888,
                            "nnz": 493692,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 19176352,
                    "linear_sparsity": 77.42222915461035,
                    "linear_total": 84934656,
                    "nnz": 43114218,
                    "pruned_heads": {
                        "0": [
                            9,
                            4,
                            5
                        ],
                        "1": [
                            0,
                            8,
                            2,
                            5
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            2,
                            3,
                            7
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            8,
                            4
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 60.40687247409585
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.37325813950282,
            "fill_rate": 0.22577770845389655,
            "speedup": 1.302586687378539
        },
        {
            "annotate": "22",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.77294228949859,
                    "f1": 87.35885990249378
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v11-a4-l20-dl1--2021-01-24--15-48-09/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 4,
                    "attention_block_rows": 4,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 4,
                    "dense_block_rows": 4,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 29.662232711791994,
                    "eval_elapsed_time": 37.211166836321354
                },
                "speedup": 1.3011290613342195,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 369024,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1492400,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1861424,
                            "linear_total": 7077888,
                            "nnz": 1870032,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 467072,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1640528,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2107600,
                            "linear_total": 7077888,
                            "nnz": 2116180,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 242352,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 173264,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 415616,
                            "linear_total": 7077888,
                            "nnz": 422044,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 157280,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 168800,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 326080,
                            "linear_total": 7077888,
                            "nnz": 332188,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 643248,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1776032,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2419280,
                            "linear_total": 7077888,
                            "nnz": 2428408,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 744560,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1758000,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2502560,
                            "linear_total": 7077888,
                            "nnz": 2511776,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 773760,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1672784,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2446544,
                            "linear_total": 7077888,
                            "nnz": 2456100,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 636208,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1581568,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2217776,
                            "linear_total": 7077888,
                            "nnz": 2226980,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 605664,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1321040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1926704,
                            "linear_total": 7077888,
                            "nnz": 1935868,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 548160,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 906384,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1454544,
                            "linear_total": 7077888,
                            "nnz": 1463048,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 486464,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 524352,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1010816,
                            "linear_total": 7077888,
                            "nnz": 1019180,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 306864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 180544,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 487408,
                            "linear_total": 7077888,
                            "nnz": 493692,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 19176352,
                    "linear_sparsity": 77.42222915461035,
                    "linear_total": 84934656,
                    "nnz": 43114218,
                    "pruned_heads": {
                        "0": [
                            9,
                            4,
                            5
                        ],
                        "1": [
                            0,
                            8,
                            2,
                            5
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            2,
                            3,
                            7
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            8,
                            4
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 60.40687247409585
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.35885990249378,
            "fill_rate": 0.22577770845389655,
            "speedup": 1.3011290613342195
        },
        {
            "annotate": "13",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.84295175023652,
                    "f1": 85.93146728512978
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v11-a4-l40-dl1--2021-01-24--15-48-35/checkpoint-105000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 4,
                    "attention_block_rows": 4,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 4,
                    "dense_block_rows": 4,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 40.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 24.637864067077636,
                    "eval_elapsed_time": 32.05906807305291
                },
                "speedup": 1.5664666750452154,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 244080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 760240,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1004320,
                            "linear_total": 7077888,
                            "nnz": 1012052,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 377328,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 887488,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1264816,
                            "linear_total": 7077888,
                            "nnz": 1272860,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 166640,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 117888,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 284528,
                            "linear_total": 7077888,
                            "nnz": 290496,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 113056,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 110240,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 223296,
                            "linear_total": 7077888,
                            "nnz": 228916,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 453680,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1009680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1463360,
                            "linear_total": 7077888,
                            "nnz": 1471780,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 549056,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 988176,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1537232,
                            "linear_total": 7077888,
                            "nnz": 1545700,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 480112,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 906608,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1386720,
                            "linear_total": 7077888,
                            "nnz": 1394928,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 397488,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 846544,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1244032,
                            "linear_total": 7077888,
                            "nnz": 1252028,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 373632,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 669920,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1043552,
                            "linear_total": 7077888,
                            "nnz": 1051416,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 393728,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 457008,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 850736,
                            "linear_total": 7077888,
                            "nnz": 857940,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 262272,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 280816,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 543088,
                            "linear_total": 7077888,
                            "nnz": 550008,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 221824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 101360,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 323184,
                            "linear_total": 7077888,
                            "nnz": 329008,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 11168864,
                    "linear_sparsity": 86.85005093798226,
                    "linear_total": 84934656,
                    "nnz": 35095854,
                    "pruned_heads": {
                        "0": [
                            9,
                            2,
                            4,
                            5
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            0,
                            6
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            4,
                            5
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 67.77038555929478
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 85.93146728512978,
            "fill_rate": 0.1314994906201774,
            "speedup": 1.5664666750452154
        },
        {
            "annotate": "13",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.47398297067171,
                    "f1": 85.88482767255138
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v11-a4-l40-dl1--2021-01-24--15-48-35/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 4,
                    "attention_block_rows": 4,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 4,
                    "dense_block_rows": 4,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 40.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 24.631753623962403,
                    "eval_elapsed_time": 32.0392144843936
                },
                "speedup": 1.5668552712310941,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 244080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 760240,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1004320,
                            "linear_total": 7077888,
                            "nnz": 1012052,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 377328,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 887488,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1264816,
                            "linear_total": 7077888,
                            "nnz": 1272860,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 166640,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 117888,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 284528,
                            "linear_total": 7077888,
                            "nnz": 290496,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 113056,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 110240,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 223296,
                            "linear_total": 7077888,
                            "nnz": 228916,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 453680,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1009680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1463360,
                            "linear_total": 7077888,
                            "nnz": 1471780,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 549056,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 988176,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1537232,
                            "linear_total": 7077888,
                            "nnz": 1545700,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 480112,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 906608,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1386720,
                            "linear_total": 7077888,
                            "nnz": 1394928,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 397488,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 846544,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1244032,
                            "linear_total": 7077888,
                            "nnz": 1252028,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 373632,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 669920,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1043552,
                            "linear_total": 7077888,
                            "nnz": 1051416,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 393728,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 457008,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 850736,
                            "linear_total": 7077888,
                            "nnz": 857940,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 262272,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 280816,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 543088,
                            "linear_total": 7077888,
                            "nnz": 550008,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 221824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 101360,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 323184,
                            "linear_total": 7077888,
                            "nnz": 329008,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 11168864,
                    "linear_sparsity": 86.85005093798226,
                    "linear_total": 84934656,
                    "nnz": 35095854,
                    "pruned_heads": {
                        "0": [
                            9,
                            2,
                            4,
                            5
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            0,
                            6
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            4,
                            5
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 67.77038555929478
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 85.88482767255138,
            "fill_rate": 0.1314994906201774,
            "speedup": 1.5668552712310941
        },
        {
            "annotate": "13",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.4077578051088,
                    "f1": 85.78500582028688
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v11-a4-l40-dl1--2021-01-24--15-48-35/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 4,
                    "attention_block_rows": 4,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 4,
                    "dense_block_rows": 4,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 40.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 24.588402084350587,
                    "eval_elapsed_time": 32.04897632403299
                },
                "speedup": 1.5696177764204813,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 244080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 760240,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1004320,
                            "linear_total": 7077888,
                            "nnz": 1012052,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 377328,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 887488,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1264816,
                            "linear_total": 7077888,
                            "nnz": 1272860,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 166640,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 117888,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 284528,
                            "linear_total": 7077888,
                            "nnz": 290496,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 113056,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 110240,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 223296,
                            "linear_total": 7077888,
                            "nnz": 228916,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 453680,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1009680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1463360,
                            "linear_total": 7077888,
                            "nnz": 1471780,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 549056,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 988176,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1537232,
                            "linear_total": 7077888,
                            "nnz": 1545700,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 480112,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 906608,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1386720,
                            "linear_total": 7077888,
                            "nnz": 1394928,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 397488,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 846544,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1244032,
                            "linear_total": 7077888,
                            "nnz": 1252028,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 373632,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 669920,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1043552,
                            "linear_total": 7077888,
                            "nnz": 1051416,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 393728,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 457008,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 850736,
                            "linear_total": 7077888,
                            "nnz": 857940,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 262272,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 280816,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 543088,
                            "linear_total": 7077888,
                            "nnz": 550008,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 221824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 101360,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 323184,
                            "linear_total": 7077888,
                            "nnz": 329008,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 11168864,
                    "linear_sparsity": 86.85005093798226,
                    "linear_total": 84934656,
                    "nnz": 35095854,
                    "pruned_heads": {
                        "0": [
                            9,
                            2,
                            4,
                            5
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            0,
                            6
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            4,
                            5
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 67.77038555929478
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 85.78500582028688,
            "fill_rate": 0.1314994906201774,
            "speedup": 1.5696177764204813
        }
    ],
    "Full block method, bs= 8x8, dl=0.25": [
        {
            "annotate": "57",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 81.10690633869442,
                    "f1": 88.3744311515211
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v9-a8-l10--2021-01-20--18-59-37/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 8,
                    "attention_block_rows": 8,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 8,
                    "dense_block_rows": 8,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 32.22343955230713,
                    "eval_elapsed_time": 39.62965265568346
                },
                "speedup": 1.1977117757004876,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 446080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4004864,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4450944,
                            "linear_total": 7077888,
                            "nnz": 4459384,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 597312,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4076928,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4674240,
                            "linear_total": 7077888,
                            "nnz": 4682824,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 362048,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1517376,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1879424,
                            "linear_total": 7077888,
                            "nnz": 1888000,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 217216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1063808,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1281024,
                            "linear_total": 7077888,
                            "nnz": 1288536,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 800192,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4155456,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4955648,
                            "linear_total": 7077888,
                            "nnz": 4964752,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 948864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4165760,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5114624,
                            "linear_total": 7077888,
                            "nnz": 5123824,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1019200,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4152640,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5171840,
                            "linear_total": 7077888,
                            "nnz": 5181392,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 915392,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4108416,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5023808,
                            "linear_total": 7077888,
                            "nnz": 5032920,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 916160,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3960384,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4876544,
                            "linear_total": 7077888,
                            "nnz": 4885872,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 834176,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3685056,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4519232,
                            "linear_total": 7077888,
                            "nnz": 4528312,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 713856,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3207936,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3921792,
                            "linear_total": 7077888,
                            "nnz": 3930904,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 465600,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2115456,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2581056,
                            "linear_total": 7077888,
                            "nnz": 2589728,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 48450176,
                    "linear_sparsity": 42.95594015239198,
                    "linear_total": 84934656,
                    "nnz": 72395170,
                    "pruned_heads": {
                        "0": [
                            9,
                            2,
                            4,
                            5
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [],
                        "5": [
                            1,
                            2,
                            6,
                            7
                        ],
                        "6": [
                            2,
                            3,
                            7
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            8,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 33.51726342179023
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.3744311515211,
            "fill_rate": 0.5704405984760802,
            "speedup": 1.1977117757004876
        },
        {
            "annotate": "57",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.82308420056765,
                    "f1": 88.21300800880684
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v9-a8-l10--2021-01-20--18-59-37/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 8,
                    "attention_block_rows": 8,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 8,
                    "dense_block_rows": 8,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 32.25489320373535,
                    "eval_elapsed_time": 39.64649308426306
                },
                "speedup": 1.1965438162077555,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 446080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4004864,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4450944,
                            "linear_total": 7077888,
                            "nnz": 4459384,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 597312,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4076928,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4674240,
                            "linear_total": 7077888,
                            "nnz": 4682824,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 362048,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1517376,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1879424,
                            "linear_total": 7077888,
                            "nnz": 1888000,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 217216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1063808,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1281024,
                            "linear_total": 7077888,
                            "nnz": 1288536,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 800192,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4155456,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4955648,
                            "linear_total": 7077888,
                            "nnz": 4964752,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 948864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4165760,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5114624,
                            "linear_total": 7077888,
                            "nnz": 5123824,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1019200,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4152640,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5171840,
                            "linear_total": 7077888,
                            "nnz": 5181392,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 915392,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4108416,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 5023808,
                            "linear_total": 7077888,
                            "nnz": 5032920,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 916160,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3960384,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4876544,
                            "linear_total": 7077888,
                            "nnz": 4885872,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 834176,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3685056,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4519232,
                            "linear_total": 7077888,
                            "nnz": 4528312,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 713856,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3207936,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3921792,
                            "linear_total": 7077888,
                            "nnz": 3930904,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 465600,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2115456,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2581056,
                            "linear_total": 7077888,
                            "nnz": 2589728,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 48450176,
                    "linear_sparsity": 42.95594015239198,
                    "linear_total": 84934656,
                    "nnz": 72395170,
                    "pruned_heads": {
                        "0": [
                            9,
                            2,
                            4,
                            5
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [],
                        "5": [
                            1,
                            2,
                            6,
                            7
                        ],
                        "6": [
                            2,
                            3,
                            7
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            8,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 33.51726342179023
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.21300800880684,
            "fill_rate": 0.5704405984760802,
            "speedup": 1.1965438162077555
        },
        {
            "annotate": "45",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.05676442762535,
                    "f1": 87.66615713942541
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v9-a8-l20--2021-01-20--19-00-06/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 8,
                    "attention_block_rows": 8,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 8,
                    "dense_block_rows": 8,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 28.86345721435547,
                    "eval_elapsed_time": 36.22357800696045
                },
                "speedup": 1.3371368758339826,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 326336,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3501120,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3827456,
                            "linear_total": 7077888,
                            "nnz": 3835760,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 487552,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3653568,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4141120,
                            "linear_total": 7077888,
                            "nnz": 4149640,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 238208,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 756608,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 994816,
                            "linear_total": 7077888,
                            "nnz": 1002184,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 141568,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 622848,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 764416,
                            "linear_total": 7077888,
                            "nnz": 771008,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 487616,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3801472,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4289088,
                            "linear_total": 7077888,
                            "nnz": 4297720,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 712832,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3800064,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4512896,
                            "linear_total": 7077888,
                            "nnz": 4521776,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 646272,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3743872,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4390144,
                            "linear_total": 7077888,
                            "nnz": 4398928,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 625600,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3691328,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4316928,
                            "linear_total": 7077888,
                            "nnz": 4325768,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 575808,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3461056,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 4036864,
                            "linear_total": 7077888,
                            "nnz": 4045744,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 579392,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3012928,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3592320,
                            "linear_total": 7077888,
                            "nnz": 3601136,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 405632,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2347776,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2753408,
                            "linear_total": 7077888,
                            "nnz": 2762136,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 317440,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1001344,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1318784,
                            "linear_total": 7077888,
                            "nnz": 1326816,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 38938240,
                    "linear_sparsity": 54.1550624517747,
                    "linear_total": 84934656,
                    "nnz": 62877338,
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            5,
                            6,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            4,
                            7,
                            8
                        ],
                        "3": [
                            2,
                            10,
                            4,
                            6
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            4,
                            5
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 42.257784614732465
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.66615713942541,
            "fill_rate": 0.45844937548225306,
            "speedup": 1.3371368758339826
        },
        {
            "annotate": "34",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.76064333017976,
                    "f1": 86.75922108224064
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v9-a8-l40--2021-01-20--19-00-35/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 8,
                    "attention_block_rows": 8,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 8,
                    "dense_block_rows": 8,
                    "dense_lambda": 0.25,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 40.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 25.933858947753908,
                    "eval_elapsed_time": 33.4375456799753
                },
                "speedup": 1.4881855061802785,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 241280,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2752704,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2993984,
                            "linear_total": 7077888,
                            "nnz": 3002184,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 379584,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2951104,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3330688,
                            "linear_total": 7077888,
                            "nnz": 3338984,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 172352,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 419008,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 591360,
                            "linear_total": 7077888,
                            "nnz": 597800,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 104768,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 388288,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 493056,
                            "linear_total": 7077888,
                            "nnz": 499016,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 322880,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3194240,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3517120,
                            "linear_total": 7077888,
                            "nnz": 3525400,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 565440,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3155136,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3720576,
                            "linear_total": 7077888,
                            "nnz": 3729376,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 390400,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 3064768,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3455168,
                            "linear_total": 7077888,
                            "nnz": 3463576,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 406592,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2993600,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3400192,
                            "linear_total": 7077888,
                            "nnz": 3408712,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 356480,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2631680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2988160,
                            "linear_total": 7077888,
                            "nnz": 2996696,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 409920,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2067776,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2477696,
                            "linear_total": 7077888,
                            "nnz": 2486200,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 242048,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1370368,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1612416,
                            "linear_total": 7077888,
                            "nnz": 1620288,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 224896,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 472768,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 697664,
                            "linear_total": 7077888,
                            "nnz": 704192,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 29278080,
                    "linear_sparsity": 65.52870008680556,
                    "linear_total": 84934656,
                    "nnz": 53211146,
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            8,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 51.13454941064908
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.75922108224064,
            "fill_rate": 0.3447129991319444,
            "speedup": 1.4881855061802785
        }
    ],
    "Full block method, bs= 8x8, dl=1.0": [
        {
            "annotate": "35",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.6244087038789,
                    "f1": 88.02730364897265
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v11-a8-l10-dl1--2021-01-24--15-46-20/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 8,
                    "attention_block_rows": 8,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 8,
                    "dense_block_rows": 8,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 29.553753234863283,
                    "eval_elapsed_time": 36.97127141384408
                },
                "speedup": 1.3059049623464731,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 633664,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2102592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2736256,
                            "linear_total": 7077888,
                            "nnz": 2744976,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 662336,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2319616,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2981952,
                            "linear_total": 7077888,
                            "nnz": 2990560,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 396032,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 297856,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 693888,
                            "linear_total": 7077888,
                            "nnz": 700408,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 262208,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 297792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 560000,
                            "linear_total": 7077888,
                            "nnz": 566320,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 975296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2636544,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3611840,
                            "linear_total": 7077888,
                            "nnz": 3621032,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1107968,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2680128,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3788096,
                            "linear_total": 7077888,
                            "nnz": 3797344,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1247936,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2623936,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3871872,
                            "linear_total": 7077888,
                            "nnz": 3881544,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1181888,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2558208,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3740096,
                            "linear_total": 7077888,
                            "nnz": 3749616,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1015040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2132480,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3147520,
                            "linear_total": 7077888,
                            "nnz": 3156728,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 913792,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1523328,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2437120,
                            "linear_total": 7077888,
                            "nnz": 2445696,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 818752,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 827264,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1646016,
                            "linear_total": 7077888,
                            "nnz": 1654216,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 514368,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 304640,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 819008,
                            "linear_total": 7077888,
                            "nnz": 825488,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 30033664,
                    "linear_sparsity": 64.6390938464506,
                    "linear_total": 84934656,
                    "nnz": 53972650,
                    "pruned_heads": {
                        "0": [
                            9,
                            4,
                            5
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            2,
                            3,
                            7
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            8,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 50.4352365996528
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.02730364897265,
            "fill_rate": 0.353609061535494,
            "speedup": 1.3059049623464731
        },
        {
            "annotate": "35",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.54872280037843,
                    "f1": 87.861684752796
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v11-a8-l10-dl1--2021-01-24--15-46-20/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 8,
                    "attention_block_rows": 8,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 8,
                    "dense_block_rows": 8,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 29.255816642761232,
                    "eval_elapsed_time": 36.84984774328768
                },
                "speedup": 1.319204091160467,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 633664,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2102592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2736256,
                            "linear_total": 7077888,
                            "nnz": 2744976,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 662336,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2319616,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2981952,
                            "linear_total": 7077888,
                            "nnz": 2990560,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 396032,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 297856,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 693888,
                            "linear_total": 7077888,
                            "nnz": 700408,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 262208,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 297792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 560000,
                            "linear_total": 7077888,
                            "nnz": 566320,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 975296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2636544,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3611840,
                            "linear_total": 7077888,
                            "nnz": 3621032,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1107968,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2680128,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3788096,
                            "linear_total": 7077888,
                            "nnz": 3797344,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1247936,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2623936,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3871872,
                            "linear_total": 7077888,
                            "nnz": 3881544,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1181888,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2558208,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3740096,
                            "linear_total": 7077888,
                            "nnz": 3749616,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1015040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2132480,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3147520,
                            "linear_total": 7077888,
                            "nnz": 3156728,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 913792,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1523328,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2437120,
                            "linear_total": 7077888,
                            "nnz": 2445696,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 818752,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 827264,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1646016,
                            "linear_total": 7077888,
                            "nnz": 1654216,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 514368,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 304640,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 819008,
                            "linear_total": 7077888,
                            "nnz": 825488,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 30033664,
                    "linear_sparsity": 64.6390938464506,
                    "linear_total": 84934656,
                    "nnz": 53972650,
                    "pruned_heads": {
                        "0": [
                            9,
                            4,
                            5
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            2,
                            3,
                            7
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            8,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 50.4352365996528
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.861684752796,
            "fill_rate": 0.353609061535494,
            "speedup": 1.319204091160467
        },
        {
            "annotate": "22",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.05392620624409,
                    "f1": 86.84949475139184
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v11-a8-l20-dl1--2021-01-24--15-46-47/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 8,
                    "attention_block_rows": 8,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 8,
                    "dense_block_rows": 8,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 24.667898628234862,
                    "eval_elapsed_time": 32.10200677579269
                },
                "speedup": 1.5645594133095706,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 407936,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1088064,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1496000,
                            "linear_total": 7077888,
                            "nnz": 1503608,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 569088,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1378944,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1948032,
                            "linear_total": 7077888,
                            "nnz": 1955776,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 298112,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 181568,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 479680,
                            "linear_total": 7077888,
                            "nnz": 485744,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 185728,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 199488,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 385216,
                            "linear_total": 7077888,
                            "nnz": 390952,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 770560,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1695552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2466112,
                            "linear_total": 7077888,
                            "nnz": 2474536,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 902848,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1680512,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2583360,
                            "linear_total": 7077888,
                            "nnz": 2591904,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 913216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1624640,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2537856,
                            "linear_total": 7077888,
                            "nnz": 2546584,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 749440,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1534912,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2284352,
                            "linear_total": 7077888,
                            "nnz": 2292792,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 684480,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1190976,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1875456,
                            "linear_total": 7077888,
                            "nnz": 1883488,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 672320,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 815872,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1488192,
                            "linear_total": 7077888,
                            "nnz": 1495544,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 570176,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 399104,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 969280,
                            "linear_total": 7077888,
                            "nnz": 976272,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 345664,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 167744,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 513408,
                            "linear_total": 7077888,
                            "nnz": 519352,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 19026944,
                    "linear_sparsity": 77.59813850308642,
                    "linear_total": 84934656,
                    "nnz": 42955274,
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            5,
                            6,
                            9
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            2
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            4
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 60.55283569350244
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.84949475139184,
            "fill_rate": 0.22401861496913578,
            "speedup": 1.5645594133095706
        },
        {
            "annotate": "13",
            "cat_fun_name": "is_full_block",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 76.87795648060549,
                    "f1": 85.16652519097626
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_aws/aws_nn-pruning-v11-a8-l40-dl1--2021-01-24--15-47-15/checkpoint-110660",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 8,
                    "attention_block_rows": 8,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 8,
                    "dense_block_rows": 8,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 40.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 19.238733966827393,
                    "eval_elapsed_time": 26.43846725206822
                },
                "speedup": 2.0060775865978457,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 330432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 520000,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 850432,
                            "linear_total": 7077888,
                            "nnz": 856864,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 468224,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 724864,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1193088,
                            "linear_total": 7077888,
                            "nnz": 1199800,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 206912,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 137088,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 344000,
                            "linear_total": 7077888,
                            "nnz": 349744,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 127744,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 128064,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 255808,
                            "linear_total": 7077888,
                            "nnz": 261080,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 511104,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 975680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1486784,
                            "linear_total": 7077888,
                            "nnz": 1493920,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 688192,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 908032,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1596224,
                            "linear_total": 7077888,
                            "nnz": 1603528,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 551360,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 863296,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1414656,
                            "linear_total": 7077888,
                            "nnz": 1421800,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 466304,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 787328,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1253632,
                            "linear_total": 7077888,
                            "nnz": 1260664,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 451840,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 695488,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1147328,
                            "linear_total": 7077888,
                            "nnz": 1154120,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 497920,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 475840,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 973760,
                            "linear_total": 7077888,
                            "nnz": 980328,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 302528,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 217600,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 520128,
                            "linear_total": 7077888,
                            "nnz": 526056,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 255168,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 91264,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 346432,
                            "linear_total": 7077888,
                            "nnz": 352056,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 11382272,
                    "linear_sparsity": 86.59878954475309,
                    "linear_total": 84934656,
                    "nnz": 35298682,
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            5,
                            6,
                            7,
                            9
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            1,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            2,
                            11,
                            6
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            3,
                            4,
                            5,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 67.5841222976064
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/opt/ml/output",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20.0,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/opt/ml/model",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/opt/ml/model",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": false,
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 85.16652519097626,
            "fill_rate": 0.13401210455246915,
            "speedup": 2.0060775865978457
        }
    ],
    "Structured pruning": [
        {
            "annotate": "26",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.88079470198676,
                    "f1": 85.81326419854291
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_it0_fw10_r-l1_rfl2.5_al0.00156_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 768,
                    "attention_block_rows": 64,
                    "attention_lambda": 0.00156,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 2.5
                },
                "speed": {
                    "cuda_eval_elapsed_time": 13.135342720031739,
                    "eval_elapsed_time": 20.267827302217484
                },
                "speedup": 2.9382098227641706,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1474560,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1376256,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2850816,
                            "linear_total": 7077888,
                            "nnz": 2857728,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1537536,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2323968,
                            "linear_total": 7077888,
                            "nnz": 2330345,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 491520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 479232,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 970752,
                            "linear_total": 7077888,
                            "nnz": 976056,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 294912,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 786432,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1081344,
                            "linear_total": 7077888,
                            "nnz": 1086720,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1843200,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3022848,
                            "linear_total": 7077888,
                            "nnz": 3029808,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 884736,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1672704,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2557440,
                            "linear_total": 7077888,
                            "nnz": 2563969,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1620480,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2210304,
                            "linear_total": 7077888,
                            "nnz": 2216479,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 393216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1675776,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2068992,
                            "linear_total": 7077888,
                            "nnz": 2075075,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 491520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1299456,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1790976,
                            "linear_total": 7077888,
                            "nnz": 1796878,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 393216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1069056,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1462272,
                            "linear_total": 7077888,
                            "nnz": 1467960,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 737280,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1523712,
                            "linear_total": 7077888,
                            "nnz": 1529568,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 196608,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 322560,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 519168,
                            "linear_total": 7077888,
                            "nnz": 524178,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 22382592,
                    "linear_sparsity": 73.6472800925926,
                    "linear_total": 84934656,
                    "nnz": 46293486,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            10,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            10,
                            11
                        ],
                        "4": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9
                        ],
                        "9": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 57.487251773494805
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 78.9120151371807,
                    "f1": 86.62567124382491
                }
            },
            "f1": 85.81326419854291,
            "fill_rate": 0.26352719907407407,
            "speedup": 2.9382098227641706
        },
        {
            "annotate": "26",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.31598864711448,
                    "f1": 86.14732314693939
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_it0_fw10_r-l1_rfl2.5_al0.00156_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-90000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 768,
                    "attention_block_rows": 64,
                    "attention_lambda": 0.00156,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 2.5
                },
                "speed": {
                    "cuda_eval_elapsed_time": 13.55481234741211,
                    "eval_elapsed_time": 20.70654077688232
                },
                "speedup": 2.847283460382213,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1474560,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1430016,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2904576,
                            "linear_total": 7077888,
                            "nnz": 2911523,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1582080,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2368512,
                            "linear_total": 7077888,
                            "nnz": 2374918,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 491520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 499200,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 990720,
                            "linear_total": 7077888,
                            "nnz": 996037,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 294912,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 812544,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1107456,
                            "linear_total": 7077888,
                            "nnz": 1112849,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1904640,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3084288,
                            "linear_total": 7077888,
                            "nnz": 3091288,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 884736,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1715712,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2600448,
                            "linear_total": 7077888,
                            "nnz": 2607005,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1654272,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2244096,
                            "linear_total": 7077888,
                            "nnz": 2250293,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 393216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1703424,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2096640,
                            "linear_total": 7077888,
                            "nnz": 2102741,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1320960,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1910784,
                            "linear_total": 7077888,
                            "nnz": 1916828,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 393216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1082880,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1476096,
                            "linear_total": 7077888,
                            "nnz": 1481793,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 748032,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1534464,
                            "linear_total": 7077888,
                            "nnz": 1540327,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 196608,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 327168,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 523776,
                            "linear_total": 7077888,
                            "nnz": 528789,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 22841856,
                    "linear_sparsity": 73.10655381944444,
                    "linear_total": 84934656,
                    "nnz": 46753113,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            10,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            10,
                            11
                        ],
                        "4": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8,
                            10
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9
                        ],
                        "9": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 57.06516200196401
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 79.27152317880795,
                    "f1": 86.82791223756466
                }
            },
            "f1": 86.14732314693939,
            "fill_rate": 0.2689344618055556,
            "speedup": 2.847283460382213
        },
        {
            "annotate": "26",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.96594134342479,
                    "f1": 85.85795020085484
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_it0_fw10_r-l1_rfl2.5_al0.00156_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-95000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 768,
                    "attention_block_rows": 64,
                    "attention_lambda": 0.00156,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 2.5
                },
                "speed": {
                    "cuda_eval_elapsed_time": 13.45258574295044,
                    "eval_elapsed_time": 20.617251713294536
                },
                "speedup": 2.868920053202093,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1474560,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1408512,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2883072,
                            "linear_total": 7077888,
                            "nnz": 2890005,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1555968,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2342400,
                            "linear_total": 7077888,
                            "nnz": 2348789,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 491520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 491520,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 983040,
                            "linear_total": 7077888,
                            "nnz": 988352,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 294912,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 798720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1093632,
                            "linear_total": 7077888,
                            "nnz": 1099016,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1875456,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3055104,
                            "linear_total": 7077888,
                            "nnz": 3062085,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 884736,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1700352,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2585088,
                            "linear_total": 7077888,
                            "nnz": 2591635,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1635840,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2225664,
                            "linear_total": 7077888,
                            "nnz": 2231849,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 393216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1689600,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2082816,
                            "linear_total": 7077888,
                            "nnz": 2088908,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1311744,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1901568,
                            "linear_total": 7077888,
                            "nnz": 1907606,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 393216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1075200,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1468416,
                            "linear_total": 7077888,
                            "nnz": 1474108,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 741888,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1528320,
                            "linear_total": 7077888,
                            "nnz": 1534179,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 196608,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 324096,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 520704,
                            "linear_total": 7077888,
                            "nnz": 525715,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 22669824,
                    "linear_sparsity": 73.30910011574075,
                    "linear_total": 84934656,
                    "nnz": 46580969,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            10,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            10,
                            11
                        ],
                        "4": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8,
                            10
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9
                        ],
                        "9": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 57.22324719197764
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 78.9593188268685,
                    "f1": 86.6751351982691
                }
            },
            "f1": 85.85795020085484,
            "fill_rate": 0.26690899884259256,
            "speedup": 2.868920053202093
        },
        {
            "annotate": "27",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.06054872280038,
                    "f1": 85.94002543374285
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_it0_fw5_r-l1_rfl2.5_al0.0001_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-105000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 768,
                    "attention_block_rows": 64,
                    "attention_lambda": 0.0001,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 5,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 2.5
                },
                "speed": {
                    "cuda_eval_elapsed_time": 13.721765300750732,
                    "eval_elapsed_time": 20.8758007818833
                },
                "speedup": 2.812640513772638,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1474560,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1499136,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2973696,
                            "linear_total": 7077888,
                            "nnz": 2980688,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1660416,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2446848,
                            "linear_total": 7077888,
                            "nnz": 2453305,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 491520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 513024,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1004544,
                            "linear_total": 7077888,
                            "nnz": 1009870,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 294912,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 898560,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1193472,
                            "linear_total": 7077888,
                            "nnz": 1198921,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1995264,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2978304,
                            "linear_total": 7077888,
                            "nnz": 2985171,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1277952,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1855488,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3133440,
                            "linear_total": 7077888,
                            "nnz": 3140472,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 491520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1724928,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2216448,
                            "linear_total": 7077888,
                            "nnz": 2222627,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 393216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1798656,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2191872,
                            "linear_total": 7077888,
                            "nnz": 2198035,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 491520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1333248,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1824768,
                            "linear_total": 7077888,
                            "nnz": 1830692,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 393216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1133568,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1526784,
                            "linear_total": 7077888,
                            "nnz": 1532514,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 798720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1585152,
                            "linear_total": 7077888,
                            "nnz": 1591048,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 196608,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 345600,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 542208,
                            "linear_total": 7077888,
                            "nnz": 547233,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 23617536,
                    "linear_sparsity": 72.19328703703704,
                    "linear_total": 84934656,
                    "nnz": 47529298,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            10,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            10
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9
                        ],
                        "9": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 56.35236717199184
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 78.99716177861873,
                    "f1": 86.83592847349966
                }
            },
            "f1": 85.94002543374285,
            "fill_rate": 0.27806712962962965,
            "speedup": 2.812640513772638
        },
        {
            "annotate": "27",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.10785241248817,
                    "f1": 86.00835164251778
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_it0_fw5_r-l1_rfl2.5_al0.0001_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 768,
                    "attention_block_rows": 64,
                    "attention_lambda": 0.0001,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 5,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 2.5
                },
                "speed": {
                    "cuda_eval_elapsed_time": 13.714137535095215,
                    "eval_elapsed_time": 20.88254290400073
                },
                "speedup": 2.8142048967064803,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1474560,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1499136,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2973696,
                            "linear_total": 7077888,
                            "nnz": 2980688,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1658880,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2445312,
                            "linear_total": 7077888,
                            "nnz": 2451768,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 491520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 513024,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1004544,
                            "linear_total": 7077888,
                            "nnz": 1009870,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 294912,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 898560,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1193472,
                            "linear_total": 7077888,
                            "nnz": 1198921,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1993728,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2976768,
                            "linear_total": 7077888,
                            "nnz": 2983634,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1277952,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1855488,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3133440,
                            "linear_total": 7077888,
                            "nnz": 3140472,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 491520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1723392,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2214912,
                            "linear_total": 7077888,
                            "nnz": 2221090,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 393216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1798656,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2191872,
                            "linear_total": 7077888,
                            "nnz": 2198035,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 491520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1331712,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1823232,
                            "linear_total": 7077888,
                            "nnz": 1829155,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 393216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1132032,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1525248,
                            "linear_total": 7077888,
                            "nnz": 1530977,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 798720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1585152,
                            "linear_total": 7077888,
                            "nnz": 1591048,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 196608,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 345600,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 542208,
                            "linear_total": 7077888,
                            "nnz": 547233,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 23609856,
                    "linear_sparsity": 72.2023292824074,
                    "linear_total": 84934656,
                    "nnz": 47521613,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            10,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            10
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9
                        ],
                        "9": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 56.35942454654601
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 78.93093661305582,
                    "f1": 86.7840779199463
                }
            },
            "f1": 86.00835164251778,
            "fill_rate": 0.27797670717592593,
            "speedup": 2.8142048967064803
        },
        {
            "annotate": "30",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.8713339640492,
                    "f1": 85.86552240887988
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_it0_fw5_r-l1_rfl2.5_al0.0001_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-75000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 768,
                    "attention_block_rows": 64,
                    "attention_lambda": 0.0001,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 5,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 2.5
                },
                "speed": {
                    "cuda_eval_elapsed_time": 14.452293647766114,
                    "eval_elapsed_time": 21.616635580081493
                },
                "speedup": 2.670468366198511,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1474560,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1777152,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3251712,
                            "linear_total": 7077888,
                            "nnz": 3258885,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 884736,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1918464,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2803200,
                            "linear_total": 7077888,
                            "nnz": 2809889,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 491520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 583680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1075200,
                            "linear_total": 7077888,
                            "nnz": 1080572,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 294912,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1009152,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1304064,
                            "linear_total": 7077888,
                            "nnz": 1309585,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1081344,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2239488,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3320832,
                            "linear_total": 7077888,
                            "nnz": 3327986,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1277952,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2075136,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3353088,
                            "linear_total": 7077888,
                            "nnz": 3360263,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1880064,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2469888,
                            "linear_total": 7077888,
                            "nnz": 2476232,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 393216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1929216,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2322432,
                            "linear_total": 7077888,
                            "nnz": 2328680,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1508352,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2098176,
                            "linear_total": 7077888,
                            "nnz": 2104342,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 393216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1248768,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1641984,
                            "linear_total": 7077888,
                            "nnz": 1647789,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 852480,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1638912,
                            "linear_total": 7077888,
                            "nnz": 1644843,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 196608,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 370176,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 566784,
                            "linear_total": 7077888,
                            "nnz": 571825,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 25846272,
                    "linear_sparsity": 69.56922743055556,
                    "linear_total": 84934656,
                    "nnz": 49759613,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            10,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            10
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8,
                            10
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9
                        ],
                        "9": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 54.304199529987116
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 78.94985808893094,
                    "f1": 86.86276298220868
                }
            },
            "f1": 85.86552240887988,
            "fill_rate": 0.3043077256944444,
            "speedup": 2.670468366198511
        },
        {
            "annotate": "27",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.78618732261117,
                    "f1": 85.70556837897196
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_it0_fw5_r-l1_rfl2.5_al0.00156_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 768,
                    "attention_block_rows": 64,
                    "attention_lambda": 0.00156,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 5,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 2.5
                },
                "speed": {
                    "cuda_eval_elapsed_time": 13.628461513519287,
                    "eval_elapsed_time": 20.942012635990977
                },
                "speedup": 2.831896539978916,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1474560,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1420800,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2895360,
                            "linear_total": 7077888,
                            "nnz": 2902301,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1740288,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2330112,
                            "linear_total": 7077888,
                            "nnz": 2336429,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 491520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 560640,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1052160,
                            "linear_total": 7077888,
                            "nnz": 1057517,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 294912,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 910848,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1205760,
                            "linear_total": 7077888,
                            "nnz": 1211217,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2061312,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3240960,
                            "linear_total": 7077888,
                            "nnz": 3248062,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1915392,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3095040,
                            "linear_total": 7077888,
                            "nnz": 3101983,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 491520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1800192,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2291712,
                            "linear_total": 7077888,
                            "nnz": 2297940,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 393216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1827840,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2221056,
                            "linear_total": 7077888,
                            "nnz": 2227238,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 491520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1370112,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1861632,
                            "linear_total": 7077888,
                            "nnz": 1867580,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 196608,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1104384,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1300992,
                            "linear_total": 7077888,
                            "nnz": 1306511,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 850944,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1637376,
                            "linear_total": 7077888,
                            "nnz": 1643306,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 196608,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 385536,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 582144,
                            "linear_total": 7077888,
                            "nnz": 587195,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 23714304,
                    "linear_sparsity": 72.07935474537037,
                    "linear_total": 84934656,
                    "nnz": 47626001,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            10,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            10
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            11
                        ],
                        "4": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            10,
                            11
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9
                        ],
                        "9": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 56.2635617989908
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 79.120151371807,
                    "f1": 86.7638780082266
                }
            },
            "f1": 85.70556837897196,
            "fill_rate": 0.2792064525462963,
            "speedup": 2.831896539978916
        },
        {
            "annotate": "20",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.70104068117313,
                    "f1": 85.6071153919288
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_bm1_it0_fw5_r-l1_rfl5_al1e-05_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 768,
                    "attention_block_rows": 64,
                    "attention_lambda": 1e-05,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 5,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 5
                },
                "speed": {
                    "cuda_eval_elapsed_time": 12.066676223754882,
                    "eval_elapsed_time": 19.25184288667515
                },
                "speedup": 3.198427826329907,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1474560,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 728064,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2202624,
                            "linear_total": 7077888,
                            "nnz": 2209114,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 393216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 998400,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1391616,
                            "linear_total": 7077888,
                            "nnz": 1397258,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 491520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 314880,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 806400,
                            "linear_total": 7077888,
                            "nnz": 811597,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 393216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 511488,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 904704,
                            "linear_total": 7077888,
                            "nnz": 909965,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1218048,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2397696,
                            "linear_total": 7077888,
                            "nnz": 2404249,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1122816,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2302464,
                            "linear_total": 7077888,
                            "nnz": 2308891,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 491520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1201152,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1692672,
                            "linear_total": 7077888,
                            "nnz": 1698510,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1204224,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1794048,
                            "linear_total": 7077888,
                            "nnz": 1800016,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 870912,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1460736,
                            "linear_total": 7077888,
                            "nnz": 1466487,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 393216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 761856,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1155072,
                            "linear_total": 7077888,
                            "nnz": 1160560,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 503808,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1290240,
                            "linear_total": 7077888,
                            "nnz": 1295944,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 196608,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 227328,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 423936,
                            "linear_total": 7077888,
                            "nnz": 428884,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 17822208,
                    "linear_sparsity": 79.0165653935185,
                    "linear_total": 84934656,
                    "nnz": 41730197,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            10,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            11
                        ],
                        "4": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            10,
                            11
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8,
                            10
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9
                        ],
                        "9": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 61.67786201057612
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 78.61873226111636,
                    "f1": 86.37059709799422
                }
            },
            "f1": 85.6071153919288,
            "fill_rate": 0.209834346064815,
            "speedup": 3.198427826329907
        },
        {
            "annotate": "16",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.94701986754967,
                    "f1": 85.90050035022541
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_it0_fw10_r-l1_rfl10.0_al0.00156_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-100000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 768,
                    "attention_block_rows": 64,
                    "attention_lambda": 0.00156,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 11.486401634216309,
                    "eval_elapsed_time": 18.590640037320554
                },
                "speedup": 3.3600072707194957,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 293376,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1669632,
                            "linear_total": 7077888,
                            "nnz": 1679616,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 491520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 422400,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 913920,
                            "linear_total": 7077888,
                            "nnz": 923904,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 118272,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 708096,
                            "linear_total": 7077888,
                            "nnz": 718080,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 393216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 225792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 619008,
                            "linear_total": 7077888,
                            "nnz": 628992,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 592896,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1969152,
                            "linear_total": 7077888,
                            "nnz": 1979136,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1081344,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 631296,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1712640,
                            "linear_total": 7077888,
                            "nnz": 1722624,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 884736,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 674304,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1559040,
                            "linear_total": 7077888,
                            "nnz": 1569024,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 629760,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1219584,
                            "linear_total": 7077888,
                            "nnz": 1229568,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 471552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1257984,
                            "linear_total": 7077888,
                            "nnz": 1267968,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 540672,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 414720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 955392,
                            "linear_total": 7077888,
                            "nnz": 965376,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 835584,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 254976,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1090560,
                            "linear_total": 7077888,
                            "nnz": 1100544,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 344064,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 87552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 431616,
                            "linear_total": 7077888,
                            "nnz": 441600,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 14106624,
                    "linear_sparsity": 83.3912037037037,
                    "linear_total": 84934656,
                    "nnz": 38065154,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            10,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            4,
                            5,
                            7,
                            8
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            10,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            10,
                            11
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            10
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9
                        ],
                        "9": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 65.04358500448319
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 78.29706717123936,
                    "f1": 86.2625032125089
                }
            },
            "f1": 85.90050035022541,
            "fill_rate": 0.16608796296296302,
            "speedup": 3.3600072707194957
        },
        {
            "annotate": "16",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.43614001892148,
                    "f1": 85.51882546766822
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_it0_fw10_r-l1_rfl10.0_al0.00156_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 768,
                    "attention_block_rows": 64,
                    "attention_lambda": 0.00156,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 11.375749713897706,
                    "eval_elapsed_time": 18.49867358384654
                },
                "speedup": 3.3926900622840255,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1425408,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 287232,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1712640,
                            "linear_total": 7077888,
                            "nnz": 1722624,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 491520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 416256,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 907776,
                            "linear_total": 7077888,
                            "nnz": 917760,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 638976,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 118272,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 757248,
                            "linear_total": 7077888,
                            "nnz": 767232,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 393216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 225792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 619008,
                            "linear_total": 7077888,
                            "nnz": 628992,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 591360,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1967616,
                            "linear_total": 7077888,
                            "nnz": 1977600,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1081344,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 629760,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1711104,
                            "linear_total": 7077888,
                            "nnz": 1721088,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 933888,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 674304,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1608192,
                            "linear_total": 7077888,
                            "nnz": 1618176,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 625152,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1214976,
                            "linear_total": 7077888,
                            "nnz": 1224960,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 688128,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 473088,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1161216,
                            "linear_total": 7077888,
                            "nnz": 1171200,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 540672,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 413184,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 953856,
                            "linear_total": 7077888,
                            "nnz": 963840,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 254976,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1041408,
                            "linear_total": 7077888,
                            "nnz": 1051392,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 393216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 89088,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 482304,
                            "linear_total": 7077888,
                            "nnz": 492288,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 14137344,
                    "linear_sparsity": 83.35503472222221,
                    "linear_total": 84934656,
                    "nnz": 38095874,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            10,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            4,
                            5,
                            7,
                            8
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            10,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            10,
                            11
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8,
                            10
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9
                        ],
                        "9": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 65.0153738728886
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 78.240302743614,
                    "f1": 86.19280466015066
                }
            },
            "f1": 85.51882546766822,
            "fill_rate": 0.1664496527777779,
            "speedup": 3.3926900622840255
        },
        {
            "annotate": "17",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.9848628192999,
                    "f1": 85.88807770994393
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_it0_fw10_r-l1_rfl10.0_al0.00156_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-90000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 768,
                    "attention_block_rows": 64,
                    "attention_lambda": 0.00156,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 11.534610126495362,
                    "eval_elapsed_time": 18.694298257119954
                },
                "speedup": 3.345964239980557,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1425408,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 314880,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1740288,
                            "linear_total": 7077888,
                            "nnz": 1750272,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 491520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 448512,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 940032,
                            "linear_total": 7077888,
                            "nnz": 950016,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 638976,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 121344,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 760320,
                            "linear_total": 7077888,
                            "nnz": 770304,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 393216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 233472,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 626688,
                            "linear_total": 7077888,
                            "nnz": 636672,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 615936,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1992192,
                            "linear_total": 7077888,
                            "nnz": 2002176,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1081344,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 646656,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1728000,
                            "linear_total": 7077888,
                            "nnz": 1737984,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 933888,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 717312,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1651200,
                            "linear_total": 7077888,
                            "nnz": 1661184,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 655872,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1245696,
                            "linear_total": 7077888,
                            "nnz": 1255680,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 482304,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1268736,
                            "linear_total": 7077888,
                            "nnz": 1278720,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 442368,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 434688,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 877056,
                            "linear_total": 7077888,
                            "nnz": 887040,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 262656,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1049088,
                            "linear_total": 7077888,
                            "nnz": 1059072,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 540672,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 89088,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 629760,
                            "linear_total": 7077888,
                            "nnz": 639744,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 14509056,
                    "linear_sparsity": 82.9173900462963,
                    "linear_total": 84934656,
                    "nnz": 38467586,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            10,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            4,
                            5,
                            7,
                            8
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            10,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            10,
                            11
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            10
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9
                        ],
                        "9": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 64.67401918059409
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 78.26868495742669,
                    "f1": 86.30683282660192
                }
            },
            "f1": 85.88807770994393,
            "fill_rate": 0.17082609953703698,
            "speedup": 3.345964239980557
        },
        {
            "annotate": "15",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 76.54683065279092,
                    "f1": 84.56290825102765
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_it0_fw10_r-l1_rfl10.0_al0.05_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-105000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 768,
                    "attention_block_rows": 64,
                    "attention_lambda": 0.05,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 10.910909118652343,
                    "eval_elapsed_time": 18.124292518012226
                },
                "speedup": 3.537229811528122,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 287232,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1663488,
                            "linear_total": 7077888,
                            "nnz": 1673472,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 835584,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 446976,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1282560,
                            "linear_total": 7077888,
                            "nnz": 1292544,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 491520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 153600,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 645120,
                            "linear_total": 7077888,
                            "nnz": 655104,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 196608,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 215040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 411648,
                            "linear_total": 7077888,
                            "nnz": 421632,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 626688,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1806336,
                            "linear_total": 7077888,
                            "nnz": 1816320,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 884736,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 566784,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1451520,
                            "linear_total": 7077888,
                            "nnz": 1461504,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 688128,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 697344,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1385472,
                            "linear_total": 7077888,
                            "nnz": 1395456,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 675840,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1265664,
                            "linear_total": 7077888,
                            "nnz": 1275648,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 479232,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1069056,
                            "linear_total": 7077888,
                            "nnz": 1079040,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 393216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 416256,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 809472,
                            "linear_total": 7077888,
                            "nnz": 819456,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 279552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1065984,
                            "linear_total": 7077888,
                            "nnz": 1075968,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 393216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 125952,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 519168,
                            "linear_total": 7077888,
                            "nnz": 529152,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 13375488,
                    "linear_sparsity": 84.25202546296296,
                    "linear_total": 84934656,
                    "nnz": 37334018,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            3,
                            10,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            8,
                            9,
                            10,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            10,
                            11
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8,
                            10
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9
                        ],
                        "9": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 65.7150099364344
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 77.96594134342479,
                    "f1": 85.91370280008687
                }
            },
            "f1": 84.56290825102765,
            "fill_rate": 0.15747974537037035,
            "speedup": 3.537229811528122
        },
        {
            "annotate": "15",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 75.99810785241249,
                    "f1": 84.26442986520863
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_it0_fw10_r-l1_rfl10.0_al0.05_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 768,
                    "attention_block_rows": 64,
                    "attention_lambda": 0.05,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 10.839029987335206,
                    "eval_elapsed_time": 17.95050869276747
                },
                "speedup": 3.5606869849485117,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 282624,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1658880,
                            "linear_total": 7077888,
                            "nnz": 1668864,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 446976,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1233408,
                            "linear_total": 7077888,
                            "nnz": 1243392,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 491520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 153600,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 645120,
                            "linear_total": 7077888,
                            "nnz": 655104,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 196608,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 215040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 411648,
                            "linear_total": 7077888,
                            "nnz": 421632,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 626688,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1806336,
                            "linear_total": 7077888,
                            "nnz": 1816320,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 568320,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1354752,
                            "linear_total": 7077888,
                            "nnz": 1364736,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 688128,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 698880,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1387008,
                            "linear_total": 7077888,
                            "nnz": 1396992,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 678912,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1268736,
                            "linear_total": 7077888,
                            "nnz": 1278720,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 480768,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1070592,
                            "linear_total": 7077888,
                            "nnz": 1080576,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 393216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 416256,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 809472,
                            "linear_total": 7077888,
                            "nnz": 819456,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 279552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1065984,
                            "linear_total": 7077888,
                            "nnz": 1075968,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 393216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 125952,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 519168,
                            "linear_total": 7077888,
                            "nnz": 529152,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 13231104,
                    "linear_sparsity": 84.42201967592592,
                    "linear_total": 84934656,
                    "nnz": 37189634,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            3,
                            10,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            10,
                            11
                        ],
                        "4": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            8,
                            9,
                            10,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            10,
                            11
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8,
                            10
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9
                        ],
                        "9": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 65.84760225492897
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 77.69157994323557,
                    "f1": 85.76954041169931
                }
            },
            "f1": 84.26442986520863,
            "fill_rate": 0.1557798032407408,
            "speedup": 3.5606869849485117
        },
        {
            "annotate": "13",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 76.3197729422895,
                    "f1": 84.62201750681498
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_it0_fw10_r-l1_rfl14.9999_al0.05_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-100000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 768,
                    "attention_block_rows": 64,
                    "attention_lambda": 0.05,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 14.9999
                },
                "speed": {
                    "cuda_eval_elapsed_time": 10.616429515838623,
                    "eval_elapsed_time": 17.745041345246136
                },
                "speedup": 3.6353458521805493,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 211968,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1588224,
                            "linear_total": 7077888,
                            "nnz": 1598208,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 638976,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 254976,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 893952,
                            "linear_total": 7077888,
                            "nnz": 903936,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 491520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 113664,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 605184,
                            "linear_total": 7077888,
                            "nnz": 615168,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 196608,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 168960,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 365568,
                            "linear_total": 7077888,
                            "nnz": 375552,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 427008,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1803264,
                            "linear_total": 7077888,
                            "nnz": 1813248,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 460800,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1247232,
                            "linear_total": 7077888,
                            "nnz": 1257216,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 737280,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 520704,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1257984,
                            "linear_total": 7077888,
                            "nnz": 1267968,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 480768,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1070592,
                            "linear_total": 7077888,
                            "nnz": 1080576,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 388608,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 978432,
                            "linear_total": 7077888,
                            "nnz": 988416,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 393216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 314880,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 708096,
                            "linear_total": 7077888,
                            "nnz": 718080,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 198144,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 984576,
                            "linear_total": 7077888,
                            "nnz": 994560,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 196608,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 92160,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 288768,
                            "linear_total": 7077888,
                            "nnz": 298752,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 11791872,
                    "linear_sparsity": 86.11653645833334,
                    "linear_total": 84934656,
                    "nnz": 35750402,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            10,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            4,
                            5,
                            7,
                            8
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            10,
                            11
                        ],
                        "4": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            8,
                            9,
                            10,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            10,
                            11
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8,
                            10
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9
                        ],
                        "9": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 67.16929377013544
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 77.36991485335857,
                    "f1": 85.60283555208089
                }
            },
            "f1": 84.62201750681498,
            "fill_rate": 0.13883463541666652,
            "speedup": 3.6353458521805493
        },
        {
            "annotate": "13",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 76.5279091769158,
                    "f1": 84.6776690586996
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_it0_fw10_r-l1_rfl14.9999_al0.05_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-105000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 768,
                    "attention_block_rows": 64,
                    "attention_lambda": 0.05,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 14.9999
                },
                "speed": {
                    "cuda_eval_elapsed_time": 10.602042137145997,
                    "eval_elapsed_time": 17.720320571679622
                },
                "speedup": 3.6402791562343726,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 211968,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1588224,
                            "linear_total": 7077888,
                            "nnz": 1598208,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 638976,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 258048,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 897024,
                            "linear_total": 7077888,
                            "nnz": 907008,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 491520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 110592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 602112,
                            "linear_total": 7077888,
                            "nnz": 612096,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 196608,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 167424,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 364032,
                            "linear_total": 7077888,
                            "nnz": 374016,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 428544,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1804800,
                            "linear_total": 7077888,
                            "nnz": 1814784,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 457728,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1244160,
                            "linear_total": 7077888,
                            "nnz": 1254144,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 688128,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 496128,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1184256,
                            "linear_total": 7077888,
                            "nnz": 1194240,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 474624,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1064448,
                            "linear_total": 7077888,
                            "nnz": 1074432,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 387072,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 976896,
                            "linear_total": 7077888,
                            "nnz": 986880,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 393216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 311808,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 705024,
                            "linear_total": 7077888,
                            "nnz": 715008,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 198144,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 984576,
                            "linear_total": 7077888,
                            "nnz": 994560,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 196608,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 92160,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 288768,
                            "linear_total": 7077888,
                            "nnz": 298752,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 11704320,
                    "linear_sparsity": 86.21961805555556,
                    "linear_total": 84934656,
                    "nnz": 35662850,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            10,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            4,
                            5,
                            7,
                            8
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            10,
                            11
                        ],
                        "4": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            8,
                            9,
                            10,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            10,
                            11
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8,
                            10
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9
                        ],
                        "9": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 67.24969549518002
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 77.41721854304636,
                    "f1": 85.51634639956605
                }
            },
            "f1": 84.6776690586996,
            "fill_rate": 0.13780381944444442,
            "speedup": 3.6402791562343726
        },
        {
            "annotate": "13",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 76.31031220435194,
                    "f1": 84.63605545666391
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_it0_fw10_r-l1_rfl14.9999_al0.05_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 768,
                    "attention_block_rows": 64,
                    "attention_lambda": 0.05,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 14.9999
                },
                "speed": {
                    "cuda_eval_elapsed_time": 10.591327346801759,
                    "eval_elapsed_time": 17.679683603346348
                },
                "speedup": 3.6439618700884893,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 210432,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1586688,
                            "linear_total": 7077888,
                            "nnz": 1596672,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 638976,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 248832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 887808,
                            "linear_total": 7077888,
                            "nnz": 897792,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 491520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 110592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 602112,
                            "linear_total": 7077888,
                            "nnz": 612096,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 196608,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 164352,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 360960,
                            "linear_total": 7077888,
                            "nnz": 370944,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 427008,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1803264,
                            "linear_total": 7077888,
                            "nnz": 1813248,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 457728,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1244160,
                            "linear_total": 7077888,
                            "nnz": 1254144,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 688128,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 486912,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1175040,
                            "linear_total": 7077888,
                            "nnz": 1185024,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 473088,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1062912,
                            "linear_total": 7077888,
                            "nnz": 1072896,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 387072,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 976896,
                            "linear_total": 7077888,
                            "nnz": 986880,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 393216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 311808,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 705024,
                            "linear_total": 7077888,
                            "nnz": 715008,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 198144,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 984576,
                            "linear_total": 7077888,
                            "nnz": 994560,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 196608,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 93696,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 290304,
                            "linear_total": 7077888,
                            "nnz": 300288,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 11679744,
                    "linear_sparsity": 86.24855324074075,
                    "linear_total": 84934656,
                    "nnz": 35638274,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            10,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            4,
                            5,
                            7,
                            8
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            10,
                            11
                        ],
                        "4": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            8,
                            9,
                            10,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            10,
                            11
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8,
                            10
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9
                        ],
                        "9": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 67.27226440045568
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 77.32261116367077,
                    "f1": 85.45260706155949
                }
            },
            "f1": 84.63605545666391,
            "fill_rate": 0.13751446759259256,
            "speedup": 3.6439618700884893
        },
        {
            "annotate": "14",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 76.75496688741723,
                    "f1": 84.83470649534952
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_it0_fw10_r-l1_rfl14.9999_al0.05_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-80000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 768,
                    "attention_block_rows": 64,
                    "attention_lambda": 0.05,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 14.9999
                },
                "speed": {
                    "cuda_eval_elapsed_time": 10.789498657226563,
                    "eval_elapsed_time": 17.89066500775516
                },
                "speedup": 3.5770330236355736,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1474560,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 247296,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1721856,
                            "linear_total": 7077888,
                            "nnz": 1731840,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 638976,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 311808,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 950784,
                            "linear_total": 7077888,
                            "nnz": 960768,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 491520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 122880,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 614400,
                            "linear_total": 7077888,
                            "nnz": 624384,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 196608,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 175104,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 371712,
                            "linear_total": 7077888,
                            "nnz": 381696,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 480768,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1857024,
                            "linear_total": 7077888,
                            "nnz": 1867008,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 491520,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1277952,
                            "linear_total": 7077888,
                            "nnz": 1287936,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 884736,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 552960,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1437696,
                            "linear_total": 7077888,
                            "nnz": 1447680,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 523776,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1113600,
                            "linear_total": 7077888,
                            "nnz": 1123584,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 425472,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1015296,
                            "linear_total": 7077888,
                            "nnz": 1025280,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 393216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 337920,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 731136,
                            "linear_total": 7077888,
                            "nnz": 741120,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 213504,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 999936,
                            "linear_total": 7077888,
                            "nnz": 1009920,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 196608,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 99840,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 296448,
                            "linear_total": 7077888,
                            "nnz": 306432,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 12387840,
                    "linear_sparsity": 85.4148582175926,
                    "linear_total": 84934656,
                    "nnz": 36346370,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            10,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            4,
                            5,
                            7,
                            8
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            10,
                            11
                        ],
                        "4": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            8,
                            9,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            10,
                            11
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8,
                            10
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9
                        ],
                        "9": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 66.62199781720042
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 77.71996215704824,
                    "f1": 85.77799129804794
                }
            },
            "f1": 84.83470649534952,
            "fill_rate": 0.14585141782407407,
            "speedup": 3.5770330236355736
        },
        {
            "annotate": "21",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.05108798486282,
                    "f1": 85.81174728555466
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_it0_fw10_r-l1_rfl5.0_al0.00156_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-105000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 768,
                    "attention_block_rows": 64,
                    "attention_lambda": 0.00156,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 5.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 12.72295495223999,
                    "eval_elapsed_time": 19.843479705043137
                },
                "speedup": 3.0334457011164853,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1474560,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 606720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2081280,
                            "linear_total": 7077888,
                            "nnz": 2091264,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 688128,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 841728,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1529856,
                            "linear_total": 7077888,
                            "nnz": 1539840,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 393216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 202752,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 595968,
                            "linear_total": 7077888,
                            "nnz": 605952,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 393216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 431616,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 824832,
                            "linear_total": 7077888,
                            "nnz": 834816,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1061376,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2437632,
                            "linear_total": 7077888,
                            "nnz": 2447616,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1081344,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1033728,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2115072,
                            "linear_total": 7077888,
                            "nnz": 2125056,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 835584,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1092096,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1927680,
                            "linear_total": 7077888,
                            "nnz": 1937664,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1101312,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1887744,
                            "linear_total": 7077888,
                            "nnz": 1897728,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 638976,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 809472,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1448448,
                            "linear_total": 7077888,
                            "nnz": 1458432,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 668160,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1257984,
                            "linear_total": 7077888,
                            "nnz": 1267968,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 391680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1178112,
                            "linear_total": 7077888,
                            "nnz": 1188096,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 491520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 173568,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 665088,
                            "linear_total": 7077888,
                            "nnz": 675072,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 17949696,
                    "linear_sparsity": 78.86646412037037,
                    "linear_total": 84934656,
                    "nnz": 41908226,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            10,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10
                        ],
                        "10": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            4,
                            5,
                            7,
                            8
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            2,
                            3,
                            4,
                            6,
                            8,
                            9,
                            10,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            8,
                            10,
                            11
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8,
                            10
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9
                        ],
                        "9": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 61.51437244200017
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 79.19583727530747,
                    "f1": 86.86229967213058
                }
            },
            "f1": 85.81174728555466,
            "fill_rate": 0.21133535879629628,
            "speedup": 3.0334457011164853
        },
        {
            "annotate": "21",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.62535477767265,
                    "f1": 85.49958980627748
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_aowd0_it0_fw10_r-l1_rfl5.0_al0.00156_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 768,
                    "attention_block_rows": 64,
                    "attention_lambda": 0.00156,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 5.0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 12.594031677246093,
                    "eval_elapsed_time": 19.77598567586392
                },
                "speedup": 3.064498644631839,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1474560,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 605184,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2079744,
                            "linear_total": 7077888,
                            "nnz": 2089728,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 840192,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1626624,
                            "linear_total": 7077888,
                            "nnz": 1636608,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 344064,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 202752,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 546816,
                            "linear_total": 7077888,
                            "nnz": 556800,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 393216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 431616,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 824832,
                            "linear_total": 7077888,
                            "nnz": 834816,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1058304,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2434560,
                            "linear_total": 7077888,
                            "nnz": 2444544,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1081344,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1035264,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2116608,
                            "linear_total": 7077888,
                            "nnz": 2126592,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 737280,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1092096,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1829376,
                            "linear_total": 7077888,
                            "nnz": 1839360,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1099776,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1886208,
                            "linear_total": 7077888,
                            "nnz": 1896192,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 688128,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 809472,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1497600,
                            "linear_total": 7077888,
                            "nnz": 1507584,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 540672,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 669696,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1210368,
                            "linear_total": 7077888,
                            "nnz": 1220352,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 391680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1178112,
                            "linear_total": 7077888,
                            "nnz": 1188096,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 175104,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 764928,
                            "linear_total": 7077888,
                            "nnz": 774912,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 17995776,
                    "linear_sparsity": 78.81221064814815,
                    "linear_total": 84934656,
                    "nnz": 41954306,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            10,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10
                        ],
                        "10": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            8,
                            9,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            4,
                            5,
                            7,
                            8
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            8,
                            9,
                            10,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            8,
                            10,
                            11
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8,
                            10
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8,
                            9
                        ],
                        "9": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 61.4720557446083
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 79.06338694418164,
                    "f1": 86.70235473718577
                }
            },
            "f1": 85.49958980627748,
            "fill_rate": 0.2118778935185185,
            "speedup": 3.064498644631839
        },
        {
            "annotate": "20",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.00378429517502,
                    "f1": 85.86131877012127
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_it0_fw10_r-l1_rfl14.9999_al0.0156_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-105000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 768,
                    "attention_block_rows": 64,
                    "attention_lambda": 0.0156,
                    "attention_output_with_dense": true,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 14.9999
                },
                "speed": {
                    "cuda_eval_elapsed_time": 14.27365227508545,
                    "eval_elapsed_time": 21.38672148110345
                },
                "speedup": 2.7038905153054142,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1459968,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 419328,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1879296,
                            "linear_total": 7077888,
                            "nnz": 1889280,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 930048,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 557568,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1487616,
                            "linear_total": 7077888,
                            "nnz": 1497600,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 413184,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 150528,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 563712,
                            "linear_total": 7077888,
                            "nnz": 573696,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 658176,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 261120,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 919296,
                            "linear_total": 7077888,
                            "nnz": 929280,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1651200,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 800256,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2451456,
                            "linear_total": 7077888,
                            "nnz": 2461440,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1181952,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 777216,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1959168,
                            "linear_total": 7077888,
                            "nnz": 1969152,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 996864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 880128,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1876992,
                            "linear_total": 7077888,
                            "nnz": 1886976,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 720384,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 886272,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1606656,
                            "linear_total": 7077888,
                            "nnz": 1616640,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 595968,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 646656,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1242624,
                            "linear_total": 7077888,
                            "nnz": 1252608,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 531456,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 494592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1026048,
                            "linear_total": 7077888,
                            "nnz": 1036032,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 1029120,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 333312,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1362432,
                            "linear_total": 7077888,
                            "nnz": 1372416,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 673536,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 110592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 784128,
                            "linear_total": 7077888,
                            "nnz": 794112,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 17159424,
                    "linear_sparsity": 79.7969111689815,
                    "linear_total": 84934656,
                    "nnz": 41117954,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            10
                        ],
                        "1": [
                            1,
                            3,
                            4,
                            6,
                            7,
                            10
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ],
                        "2": [
                            11,
                            5
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            10
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            11
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8,
                            10
                        ],
                        "7": [
                            0,
                            1,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "8": [
                            1,
                            2,
                            3,
                            4,
                            7,
                            8
                        ],
                        "9": [
                            0,
                            1,
                            3,
                            4,
                            6,
                            7,
                            8,
                            9
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 62.240103802270966
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 78.6565752128666,
                    "f1": 86.45517515140308
                }
            },
            "f1": 85.86131877012127,
            "fill_rate": 0.202030888310185,
            "speedup": 2.7038905153054142
        },
        {
            "annotate": "20",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.04162724692526,
                    "f1": 85.89832211406967
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_it0_fw10_r-l1_rfl14.9999_al0.0156_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 768,
                    "attention_block_rows": 64,
                    "attention_lambda": 0.0156,
                    "attention_output_with_dense": true,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 14.9999
                },
                "speed": {
                    "cuda_eval_elapsed_time": 14.245073196411132,
                    "eval_elapsed_time": 21.39737162971869
                },
                "speedup": 2.7093151767794685,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1460736,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 420864,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1881600,
                            "linear_total": 7077888,
                            "nnz": 1891584,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 930816,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 557568,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1488384,
                            "linear_total": 7077888,
                            "nnz": 1498368,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 413184,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 148992,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 562176,
                            "linear_total": 7077888,
                            "nnz": 572160,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 658176,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 261120,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 919296,
                            "linear_total": 7077888,
                            "nnz": 929280,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1636608,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 794112,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2430720,
                            "linear_total": 7077888,
                            "nnz": 2440704,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1172736,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 780288,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1953024,
                            "linear_total": 7077888,
                            "nnz": 1963008,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 946944,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 875520,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1822464,
                            "linear_total": 7077888,
                            "nnz": 1832448,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 719616,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 883200,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1602816,
                            "linear_total": 7077888,
                            "nnz": 1612800,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 602112,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 646656,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1248768,
                            "linear_total": 7077888,
                            "nnz": 1258752,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 530688,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 493056,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1023744,
                            "linear_total": 7077888,
                            "nnz": 1033728,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 1026816,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 333312,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1360128,
                            "linear_total": 7077888,
                            "nnz": 1370112,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 675072,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 110592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 785664,
                            "linear_total": 7077888,
                            "nnz": 795648,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 17078784,
                    "linear_sparsity": 79.89185474537037,
                    "linear_total": 84934656,
                    "nnz": 41037314,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            10
                        ],
                        "1": [
                            1,
                            3,
                            4,
                            6,
                            7,
                            10
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ],
                        "2": [
                            11,
                            5
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            10
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            11
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8,
                            10
                        ],
                        "7": [
                            0,
                            1,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "8": [
                            1,
                            2,
                            3,
                            4,
                            7,
                            8
                        ],
                        "9": [
                            0,
                            1,
                            3,
                            4,
                            6,
                            7,
                            8,
                            9
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 62.31415802270676
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 78.52412488174078,
                    "f1": 86.2110202537131
                }
            },
            "f1": 85.89832211406967,
            "fill_rate": 0.20108145254629628,
            "speedup": 2.7093151767794685
        },
        {
            "annotate": "18",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 75.67644276253547,
                    "f1": 84.4740049617883
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_it0_fw10_r-l1_rfl14.9999_al0.05_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-100000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 768,
                    "attention_block_rows": 64,
                    "attention_lambda": 0.05,
                    "attention_output_with_dense": true,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 14.9999
                },
                "speed": {
                    "cuda_eval_elapsed_time": 12.679624610900879,
                    "eval_elapsed_time": 19.804423895198852
                },
                "speedup": 3.043811957350131,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1331712,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 466944,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1798656,
                            "linear_total": 7077888,
                            "nnz": 1808640,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 473088,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 649728,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1122816,
                            "linear_total": 7077888,
                            "nnz": 1132800,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 417792,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 165888,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 583680,
                            "linear_total": 7077888,
                            "nnz": 593664,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 258048,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 301056,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 559104,
                            "linear_total": 7077888,
                            "nnz": 569088,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1500672,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 824832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2325504,
                            "linear_total": 7077888,
                            "nnz": 2335488,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 956160,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 834048,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1790208,
                            "linear_total": 7077888,
                            "nnz": 1800192,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 963840,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 923136,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1886976,
                            "linear_total": 7077888,
                            "nnz": 1896960,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 598272,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 924672,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1522944,
                            "linear_total": 7077888,
                            "nnz": 1532928,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 558336,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 700416,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1258752,
                            "linear_total": 7077888,
                            "nnz": 1268736,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 235008,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 502272,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 737280,
                            "linear_total": 7077888,
                            "nnz": 747264,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 903936,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 382464,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1286400,
                            "linear_total": 7077888,
                            "nnz": 1296384,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 533760,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 132096,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 665856,
                            "linear_total": 7077888,
                            "nnz": 675840,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 15538176,
                    "linear_sparsity": 81.70572916666666,
                    "linear_total": 84934656,
                    "nnz": 39496706,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            10
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            10
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11
                        ],
                        "2": [
                            11,
                            5,
                            7
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            11
                        ],
                        "4": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            10
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            10,
                            11
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8,
                            10
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8
                        ],
                        "9": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 63.72894627217538
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 77.67265846736045,
                    "f1": 85.79872940903662
                }
            },
            "f1": 84.4740049617883,
            "fill_rate": 0.18294270833333348,
            "speedup": 3.043811957350131
        },
        {
            "annotate": "18",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 76.0170293282876,
                    "f1": 84.48208063503463
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_it0_fw10_r-l1_rfl14.9999_al0.05_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-105000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 768,
                    "attention_block_rows": 64,
                    "attention_lambda": 0.05,
                    "attention_output_with_dense": true,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 14.9999
                },
                "speed": {
                    "cuda_eval_elapsed_time": 12.68363911819458,
                    "eval_elapsed_time": 19.829505565110594
                },
                "speedup": 3.0428485583453524,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1334784,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 465408,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1800192,
                            "linear_total": 7077888,
                            "nnz": 1810176,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 473856,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 645120,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1118976,
                            "linear_total": 7077888,
                            "nnz": 1128960,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 370176,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 165888,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 536064,
                            "linear_total": 7077888,
                            "nnz": 546048,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 257280,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 297984,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 555264,
                            "linear_total": 7077888,
                            "nnz": 565248,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1497600,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 823296,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2320896,
                            "linear_total": 7077888,
                            "nnz": 2330880,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 956160,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 832512,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1788672,
                            "linear_total": 7077888,
                            "nnz": 1798656,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 965376,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 926208,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1891584,
                            "linear_total": 7077888,
                            "nnz": 1901568,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 600576,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 920064,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1520640,
                            "linear_total": 7077888,
                            "nnz": 1530624,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 561408,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 697344,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1258752,
                            "linear_total": 7077888,
                            "nnz": 1268736,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 230400,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 502272,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 732672,
                            "linear_total": 7077888,
                            "nnz": 742656,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 897792,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 379392,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1277184,
                            "linear_total": 7077888,
                            "nnz": 1287168,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 528384,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 132096,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 660480,
                            "linear_total": 7077888,
                            "nnz": 670464,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 15461376,
                    "linear_sparsity": 81.79615162037037,
                    "linear_total": 84934656,
                    "nnz": 39419906,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            10
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            10
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11
                        ],
                        "2": [
                            11,
                            5,
                            7
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            11
                        ],
                        "4": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            10
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            10,
                            11
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8,
                            10
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8
                        ],
                        "9": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 63.79947410116185
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 77.68211920529801,
                    "f1": 85.79291904118423
                }
            },
            "f1": 84.48208063503463,
            "fill_rate": 0.18203848379629628,
            "speedup": 3.0428485583453524
        },
        {
            "annotate": "18",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 75.93188268684958,
                    "f1": 84.50981123274157
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_it0_fw10_r-l1_rfl14.9999_al0.05_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 768,
                    "attention_block_rows": 64,
                    "attention_lambda": 0.05,
                    "attention_output_with_dense": true,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 14.9999
                },
                "speed": {
                    "cuda_eval_elapsed_time": 12.51708829498291,
                    "eval_elapsed_time": 19.654158322140574
                },
                "speedup": 3.08333632357938,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1323264,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 470016,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1793280,
                            "linear_total": 7077888,
                            "nnz": 1803264,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 470016,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 645120,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1115136,
                            "linear_total": 7077888,
                            "nnz": 1125120,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 370176,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 165888,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 536064,
                            "linear_total": 7077888,
                            "nnz": 546048,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 258048,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 297984,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 556032,
                            "linear_total": 7077888,
                            "nnz": 566016,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1496832,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 824832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2321664,
                            "linear_total": 7077888,
                            "nnz": 2331648,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 960000,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 829440,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1789440,
                            "linear_total": 7077888,
                            "nnz": 1799424,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 917760,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 926208,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1843968,
                            "linear_total": 7077888,
                            "nnz": 1853952,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 607488,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 918528,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1526016,
                            "linear_total": 7077888,
                            "nnz": 1536000,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 567552,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 697344,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1264896,
                            "linear_total": 7077888,
                            "nnz": 1274880,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 231168,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 499200,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 730368,
                            "linear_total": 7077888,
                            "nnz": 740352,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 900096,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 379392,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1279488,
                            "linear_total": 7077888,
                            "nnz": 1289472,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 533760,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 133632,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 667392,
                            "linear_total": 7077888,
                            "nnz": 677376,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 15423744,
                    "linear_sparsity": 81.84045862268519,
                    "linear_total": 84934656,
                    "nnz": 39382274,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            10
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            10
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11
                        ],
                        "2": [
                            11,
                            5,
                            7
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            11
                        ],
                        "4": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            9,
                            10
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            10,
                            11
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8,
                            10
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8
                        ],
                        "9": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 63.83403273736522
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 77.4550614947966,
                    "f1": 85.48800663360532
                }
            },
            "f1": 84.50981123274157,
            "fill_rate": 0.18159541377314814,
            "speedup": 3.08333632357938
        },
        {
            "annotate": "19",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 76.13055818353831,
                    "f1": 84.59415607632204
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_it0_fw10_r-l1_rfl14.9999_al0.05_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-75000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 768,
                    "attention_block_rows": 64,
                    "attention_lambda": 0.05,
                    "attention_output_with_dense": true,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 14.9999
                },
                "speed": {
                    "cuda_eval_elapsed_time": 13.145666324615478,
                    "eval_elapsed_time": 20.2730004908517
                },
                "speedup": 2.935902376671653,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1390080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 540672,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1930752,
                            "linear_total": 7077888,
                            "nnz": 1940736,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 622848,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 724992,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1347840,
                            "linear_total": 7077888,
                            "nnz": 1357824,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 322560,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 184320,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 506880,
                            "linear_total": 7077888,
                            "nnz": 516864,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 412416,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 348672,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 761088,
                            "linear_total": 7077888,
                            "nnz": 771072,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1506816,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 916992,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2423808,
                            "linear_total": 7077888,
                            "nnz": 2433792,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 966144,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 898560,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1864704,
                            "linear_total": 7077888,
                            "nnz": 1874688,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 965376,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 990720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1956096,
                            "linear_total": 7077888,
                            "nnz": 1966080,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 734976,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1007616,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1742592,
                            "linear_total": 7077888,
                            "nnz": 1752576,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 561408,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 761856,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1323264,
                            "linear_total": 7077888,
                            "nnz": 1333248,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 282624,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 552960,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 835584,
                            "linear_total": 7077888,
                            "nnz": 845568,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 903936,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 403968,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1307904,
                            "linear_total": 7077888,
                            "nnz": 1317888,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 536064,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 145920,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 681984,
                            "linear_total": 7077888,
                            "nnz": 691968,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 16682496,
                    "linear_sparsity": 80.3584346064815,
                    "linear_total": 84934656,
                    "nnz": 40641026,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            10
                        ],
                        "1": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            10
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11
                        ],
                        "2": [
                            11,
                            5,
                            7
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            11
                        ],
                        "4": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            10
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            11
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8,
                            10
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8
                        ],
                        "9": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 62.67808162027695
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 78.25922421948913,
                    "f1": 86.10555694769658
                }
            },
            "f1": 84.59415607632204,
            "fill_rate": 0.196415653935185,
            "speedup": 2.935902376671653
        },
        {
            "annotate": "18",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 76.20624408703878,
                    "f1": 84.78885528858153
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_it0_fw10_r-l1_rfl14.9999_al0.05_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-95000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 768,
                    "attention_block_rows": 64,
                    "attention_lambda": 0.05,
                    "attention_output_with_dense": true,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 14.9999
                },
                "speed": {
                    "cuda_eval_elapsed_time": 12.730052349090576,
                    "eval_elapsed_time": 19.889838815666735
                },
                "speedup": 3.0317544615696925,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1331712,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 470016,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1801728,
                            "linear_total": 7077888,
                            "nnz": 1811712,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 471552,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 655872,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1127424,
                            "linear_total": 7077888,
                            "nnz": 1137408,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 319488,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 168960,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 488448,
                            "linear_total": 7077888,
                            "nnz": 498432,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 258048,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 302592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 560640,
                            "linear_total": 7077888,
                            "nnz": 570624,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1507584,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 834048,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2341632,
                            "linear_total": 7077888,
                            "nnz": 2351616,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 960768,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 843264,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1804032,
                            "linear_total": 7077888,
                            "nnz": 1814016,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 968448,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 930816,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1899264,
                            "linear_total": 7077888,
                            "nnz": 1909248,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 598272,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 930816,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1529088,
                            "linear_total": 7077888,
                            "nnz": 1539072,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 564480,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 700416,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1264896,
                            "linear_total": 7077888,
                            "nnz": 1274880,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 231168,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 506880,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 738048,
                            "linear_total": 7077888,
                            "nnz": 748032,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 893952,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 384000,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1277952,
                            "linear_total": 7077888,
                            "nnz": 1287936,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 535296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 133632,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 668928,
                            "linear_total": 7077888,
                            "nnz": 678912,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 15502080,
                    "linear_sparsity": 81.7482277199074,
                    "linear_total": 84934656,
                    "nnz": 39460610,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            10
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            10
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11
                        ],
                        "2": [
                            11,
                            5,
                            7
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            11
                        ],
                        "4": [
                            1,
                            2,
                            3,
                            4,
                            8,
                            9,
                            10
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            8,
                            10,
                            11
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8,
                            10
                        ],
                        "7": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            7,
                            8
                        ],
                        "9": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 63.76209435179903
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 78.00378429517502,
                    "f1": 86.00102110548276
                }
            },
            "f1": 84.78885528858153,
            "fill_rate": 0.18251772280092593,
            "speedup": 3.0317544615696925
        }
    ],
    "full block version 0": [
        {
            "annotate": "",
            "cat_fun_name": "is_block_unstructured",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.06717123935667,
                    "f1": 85.30981160352648
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold_apme-sigmoied_threshold_abr32_abc32_it0_fw10_r-l1_rfl75_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 75
                },
                "speed": {
                    "cuda_eval_elapsed_time": 24.64119245147705,
                    "eval_elapsed_time": 32.04050999786705
                },
                "speedup": 1.5662550861274434,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 36794,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1688900,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1725694,
                            "linear_total": 7077888,
                            "nnz": 1735678,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 233028,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1726592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1959620,
                            "linear_total": 7077888,
                            "nnz": 1969604,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 89475,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 329600,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 419075,
                            "linear_total": 7077888,
                            "nnz": 429059,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 45791,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 148125,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 193916,
                            "linear_total": 7077888,
                            "nnz": 203900,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 194318,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1774807,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1969125,
                            "linear_total": 7077888,
                            "nnz": 1979109,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 270153,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1742205,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2012358,
                            "linear_total": 7077888,
                            "nnz": 2022342,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 207935,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1652927,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1860862,
                            "linear_total": 7077888,
                            "nnz": 1870846,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 215427,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1599761,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1815188,
                            "linear_total": 7077888,
                            "nnz": 1825172,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 114563,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1404415,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1518978,
                            "linear_total": 7077888,
                            "nnz": 1528962,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 165011,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1142635,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1307646,
                            "linear_total": 7077888,
                            "nnz": 1317630,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 86589,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 859553,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 946142,
                            "linear_total": 7077888,
                            "nnz": 956126,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 110020,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 421789,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 531809,
                            "linear_total": 7077888,
                            "nnz": 541793,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 16260413,
                    "linear_sparsity": 80.85538487375518,
                    "linear_total": 84934656,
                    "nnz": 40218943,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "3": [
                            2,
                            3,
                            4,
                            6,
                            7,
                            8,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            8,
                            9,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "9": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10,
                            11
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 63.065693568741764
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 77.41721854304636,
                    "f1": 85.55066476449066
                }
            },
            "f1": 85.30981160352648,
            "fill_rate": 0.19144615126244813,
            "speedup": 1.5662550861274434
        }
    ],
    "improved soft movement with distillation": [
        {
            "annotate": "6",
            "cat_fun_name": "is_improved_mvmt_pruning",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.4484389782403,
                    "f1": 86.3547925481507
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold_apme-sigmoied_threshold_abr1_abc1_it0_fw10_r-l1_rfl150_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 150
                },
                "speed": {
                    "cuda_eval_elapsed_time": 29.783737594604492,
                    "eval_elapsed_time": 37.12324417894706
                },
                "speedup": 1.2958210124830911,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 30729,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 624455,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 655184,
                            "linear_total": 7077888,
                            "nnz": 665168,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 77742,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 655389,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 733131,
                            "linear_total": 7077888,
                            "nnz": 743115,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 27892,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 61389,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 89281,
                            "linear_total": 7077888,
                            "nnz": 99265,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 20781,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 51322,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 72103,
                            "linear_total": 7077888,
                            "nnz": 82087,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 70206,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 660173,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 730379,
                            "linear_total": 7077888,
                            "nnz": 740363,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 106339,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 628112,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 734451,
                            "linear_total": 7077888,
                            "nnz": 744435,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 81845,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 574018,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 655863,
                            "linear_total": 7077888,
                            "nnz": 665847,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 68554,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 537752,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 606306,
                            "linear_total": 7077888,
                            "nnz": 616290,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 58217,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 434629,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 492846,
                            "linear_total": 7077888,
                            "nnz": 502830,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 65705,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 313684,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 379389,
                            "linear_total": 7077888,
                            "nnz": 389373,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 39483,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 203724,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 243207,
                            "linear_total": 7077888,
                            "nnz": 253191,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 46007,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 73599,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 119606,
                            "linear_total": 7077888,
                            "nnz": 129590,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 5511746,
                    "linear_sparsity": 93.51060419906804,
                    "linear_total": 84934656,
                    "nnz": 29470276,
                    "pruned_heads": {
                        "0": [
                            9
                        ],
                        "1": [
                            0,
                            8,
                            2
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            9
                        ],
                        "11": [
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [],
                        "5": [
                            1,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            10,
                            2,
                            3
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 72.93652882926945
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.3547925481507,
            "fill_rate": 0.06489395800931963,
            "speedup": 1.2958210124830911
        },
        {
            "annotate": "4",
            "cat_fun_name": "is_improved_mvmt_pruning",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 77.39829706717124,
                    "f1": 85.66626983371626
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold_apme-sigmoied_threshold_abr1_abc1_it0_fw10_r-l1_rfl225_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 225
                },
                "speed": {
                    "cuda_eval_elapsed_time": 27.713626304626466,
                    "eval_elapsed_time": 35.06419681990519
                },
                "speedup": 1.3926143255719736,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 18728,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 446655,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 465383,
                            "linear_total": 7077888,
                            "nnz": 475367,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 63059,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 464338,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 527397,
                            "linear_total": 7077888,
                            "nnz": 537381,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 21311,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 43332,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 64643,
                            "linear_total": 7077888,
                            "nnz": 74627,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 17233,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 36806,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 54039,
                            "linear_total": 7077888,
                            "nnz": 64023,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 53761,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 462731,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 516492,
                            "linear_total": 7077888,
                            "nnz": 526476,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 84624,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 430348,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 514972,
                            "linear_total": 7077888,
                            "nnz": 524956,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 58345,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 384869,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 443214,
                            "linear_total": 7077888,
                            "nnz": 453198,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 50615,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 346306,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 396921,
                            "linear_total": 7077888,
                            "nnz": 406905,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 41344,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 277660,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 319004,
                            "linear_total": 7077888,
                            "nnz": 328988,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 47420,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 201763,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 249183,
                            "linear_total": 7077888,
                            "nnz": 259167,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 27562,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 133500,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 161062,
                            "linear_total": 7077888,
                            "nnz": 171046,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 34151,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 47554,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 81705,
                            "linear_total": 7077888,
                            "nnz": 91689,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 3794015,
                    "linear_sparsity": 95.5330189363456,
                    "linear_total": 84934656,
                    "nnz": 27752545,
                    "pruned_heads": {
                        "0": [
                            9,
                            2
                        ],
                        "1": [
                            8,
                            2
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            9
                        ],
                        "11": [
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            6,
                            7
                        ],
                        "5": [
                            1,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            10,
                            2,
                            3
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            4,
                            5,
                            6
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 74.51397463933142
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 85.66626983371626,
            "fill_rate": 0.04466981063654396,
            "speedup": 1.3926143255719736
        },
        {
            "annotate": "25",
            "cat_fun_name": "is_improved_mvmt_pruning",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 81.40018921475875,
                    "f1": 88.66263407974378
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold_apme-sigmoied_threshold_abr1_abc1_it0_fw10_r-l1_rfl25_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 25
                },
                "speed": {
                    "cuda_eval_elapsed_time": 37.63941863250732,
                    "eval_elapsed_time": 44.979358388110995
                },
                "speedup": 1.0253716557683228,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 158912,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1993831,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2152743,
                            "linear_total": 7077888,
                            "nnz": 2162727,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 234395,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2030737,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2265132,
                            "linear_total": 7077888,
                            "nnz": 2275116,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 134277,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 440264,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 574541,
                            "linear_total": 7077888,
                            "nnz": 584525,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 63309,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 269756,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 333065,
                            "linear_total": 7077888,
                            "nnz": 343049,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 301048,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2114464,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2415512,
                            "linear_total": 7077888,
                            "nnz": 2425496,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 358791,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2106776,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2465567,
                            "linear_total": 7077888,
                            "nnz": 2475551,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 398673,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2058594,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2457267,
                            "linear_total": 7077888,
                            "nnz": 2467251,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 367333,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2043244,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2410577,
                            "linear_total": 7077888,
                            "nnz": 2420561,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 344288,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1862492,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2206780,
                            "linear_total": 7077888,
                            "nnz": 2216764,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 304514,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1514517,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1819031,
                            "linear_total": 7077888,
                            "nnz": 1829015,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 265513,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1099308,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1364821,
                            "linear_total": 7077888,
                            "nnz": 1374805,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 201714,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 627276,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 828990,
                            "linear_total": 7077888,
                            "nnz": 838974,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 21294026,
                    "linear_sparsity": 74.92893124804085,
                    "linear_total": 84934656,
                    "nnz": 45252556,
                    "pruned_heads": {
                        "0": [],
                        "1": [],
                        "10": [
                            1
                        ],
                        "11": [
                            8,
                            5,
                            7
                        ],
                        "2": [],
                        "3": [],
                        "4": [],
                        "5": [],
                        "6": [],
                        "7": [
                            6
                        ],
                        "8": [],
                        "9": [
                            1,
                            4,
                            7
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 58.4431701722824
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.66263407974378,
            "fill_rate": 0.25071068751959147,
            "speedup": 1.0253716557683228
        },
        {
            "annotate": "3",
            "cat_fun_name": "is_improved_mvmt_pruning",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 76.98202459791864,
                    "f1": 85.40699359564026
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold_apme-sigmoied_threshold_abr1_abc1_it0_fw10_r-l1_rfl300_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 300
                },
                "speed": {
                    "cuda_eval_elapsed_time": 25.440285942077637,
                    "eval_elapsed_time": 32.748252402991056
                },
                "speedup": 1.5170581452285046,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 13195,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 344662,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 357857,
                            "linear_total": 7077888,
                            "nnz": 367841,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 53357,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 352125,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 405482,
                            "linear_total": 7077888,
                            "nnz": 415466,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 18747,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 34723,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 53470,
                            "linear_total": 7077888,
                            "nnz": 63454,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 15957,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 30412,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 46369,
                            "linear_total": 7077888,
                            "nnz": 56353,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 43981,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 351138,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 395119,
                            "linear_total": 7077888,
                            "nnz": 405103,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 71058,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 323059,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 394117,
                            "linear_total": 7077888,
                            "nnz": 404101,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 47705,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 287668,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 335373,
                            "linear_total": 7077888,
                            "nnz": 345357,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 40348,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 252178,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 292526,
                            "linear_total": 7077888,
                            "nnz": 302510,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 33002,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 205112,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 238114,
                            "linear_total": 7077888,
                            "nnz": 248098,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 38753,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 150138,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 188891,
                            "linear_total": 7077888,
                            "nnz": 198875,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 22052,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 101313,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 123365,
                            "linear_total": 7077888,
                            "nnz": 133349,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 28498,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 35917,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 64415,
                            "linear_total": 7077888,
                            "nnz": 74399,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 2895098,
                    "linear_sparsity": 96.59138196780358,
                    "linear_total": 84934656,
                    "nnz": 26853628,
                    "pruned_heads": {
                        "0": [
                            9,
                            2
                        ],
                        "1": [
                            0,
                            8,
                            2,
                            6
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            8,
                            9
                        ],
                        "11": [
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            0,
                            11,
                            6,
                            7
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            10,
                            2,
                            3
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 75.33947808267818
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 85.40699359564026,
            "fill_rate": 0.0340861803219642,
            "speedup": 1.5170581452285046
        },
        {
            "annotate": "15",
            "cat_fun_name": "is_improved_mvmt_pruning",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.53926206244087,
                    "f1": 88.07603620459668
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold_apme-sigmoied_threshold_abr1_abc1_it0_fw10_r-l1_rfl50_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 50
                },
                "speed": {
                    "cuda_eval_elapsed_time": 35.31425653076172,
                    "eval_elapsed_time": 42.675803440622985
                },
                "speedup": 1.092884200230921,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 79341,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1362813,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1442154,
                            "linear_total": 7077888,
                            "nnz": 1452138,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 146964,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1411011,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1557975,
                            "linear_total": 7077888,
                            "nnz": 1567959,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 70746,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 191871,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 262617,
                            "linear_total": 7077888,
                            "nnz": 272601,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 36271,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 137408,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 173679,
                            "linear_total": 7077888,
                            "nnz": 183663,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 173655,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1463754,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1637409,
                            "linear_total": 7077888,
                            "nnz": 1647393,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 213353,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1442359,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1655712,
                            "linear_total": 7077888,
                            "nnz": 1665696,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 221518,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1380230,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1601748,
                            "linear_total": 7077888,
                            "nnz": 1611732,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 179373,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1360274,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1539647,
                            "linear_total": 7077888,
                            "nnz": 1549631,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 168393,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1183896,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1352289,
                            "linear_total": 7077888,
                            "nnz": 1362273,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 159612,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 906603,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1066215,
                            "linear_total": 7077888,
                            "nnz": 1076199,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 127230,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 600693,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 727923,
                            "linear_total": 7077888,
                            "nnz": 737907,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 105257,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 285690,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 390947,
                            "linear_total": 7077888,
                            "nnz": 400931,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 13408315,
                    "linear_sparsity": 84.21337575088313,
                    "linear_total": 84934656,
                    "nnz": 37366845,
                    "pruned_heads": {
                        "0": [],
                        "1": [],
                        "10": [
                            8,
                            1,
                            4
                        ],
                        "11": [
                            3,
                            5,
                            7,
                            8,
                            11
                        ],
                        "2": [
                            8
                        ],
                        "3": [
                            2,
                            6
                        ],
                        "4": [],
                        "5": [
                            1
                        ],
                        "6": [
                            3
                        ],
                        "7": [
                            3,
                            6
                        ],
                        "8": [
                            0,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 65.68486388119823
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.07603620459668,
            "fill_rate": 0.15786624249116865,
            "speedup": 1.092884200230921
        },
        {
            "annotate": "16",
            "cat_fun_name": "is_improved_mvmt_pruning",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.22705771050141,
                    "f1": 88.08154392563726
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold_apme-sigmoied_threshold_abr1_abc1_it0_fw10_r-l1_rfl50_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-95000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 50
                },
                "speed": {
                    "cuda_eval_elapsed_time": 35.30916271209717,
                    "eval_elapsed_time": 42.719326278194785
                },
                "speedup": 1.0930418633843273,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 87221,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1434572,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1521793,
                            "linear_total": 7077888,
                            "nnz": 1531777,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 157517,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1480327,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1637844,
                            "linear_total": 7077888,
                            "nnz": 1647828,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 75446,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 204546,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 279992,
                            "linear_total": 7077888,
                            "nnz": 289976,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 38439,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 144390,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 182829,
                            "linear_total": 7077888,
                            "nnz": 192813,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 188172,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1535574,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1723746,
                            "linear_total": 7077888,
                            "nnz": 1733730,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 230341,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1512620,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1742961,
                            "linear_total": 7077888,
                            "nnz": 1752945,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 240387,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1447041,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1687428,
                            "linear_total": 7077888,
                            "nnz": 1697412,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 195780,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1427597,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1623377,
                            "linear_total": 7077888,
                            "nnz": 1633361,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 184963,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1245019,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1429982,
                            "linear_total": 7077888,
                            "nnz": 1439966,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 172954,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 957245,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1130199,
                            "linear_total": 7077888,
                            "nnz": 1140183,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 138133,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 635763,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 773896,
                            "linear_total": 7077888,
                            "nnz": 783880,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 112972,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 304891,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 417863,
                            "linear_total": 7077888,
                            "nnz": 427847,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 14151910,
                    "linear_sparsity": 83.3378850677867,
                    "linear_total": 84934656,
                    "nnz": 38110440,
                    "pruned_heads": {
                        "0": [],
                        "1": [],
                        "10": [
                            8,
                            1,
                            4
                        ],
                        "11": [
                            3,
                            5,
                            7,
                            8,
                            11
                        ],
                        "2": [
                            8
                        ],
                        "3": [
                            2,
                            6
                        ],
                        "4": [],
                        "5": [
                            1
                        ],
                        "6": [
                            3
                        ],
                        "7": [
                            3,
                            6
                        ],
                        "8": [
                            0,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 65.00199746198996
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.08154392563726,
            "fill_rate": 0.166621149322133,
            "speedup": 1.0930418633843273
        },
        {
            "annotate": "11",
            "cat_fun_name": "is_improved_mvmt_pruning",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.89593188268685,
                    "f1": 87.64967103979136
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold_apme-sigmoied_threshold_abr1_abc1_it0_fw10_r-l1_rfl75_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-105000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 75
                },
                "speed": {
                    "cuda_eval_elapsed_time": 32.98558323669434,
                    "eval_elapsed_time": 40.38167083170265
                },
                "speedup": 1.170038217254783,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 56754,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1054479,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1111233,
                            "linear_total": 7077888,
                            "nnz": 1121217,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 116764,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1106103,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1222867,
                            "linear_total": 7077888,
                            "nnz": 1232851,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 50915,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 121878,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 172793,
                            "linear_total": 7077888,
                            "nnz": 182777,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 28303,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 94314,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 122617,
                            "linear_total": 7077888,
                            "nnz": 132601,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 127558,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1136881,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1264439,
                            "linear_total": 7077888,
                            "nnz": 1274423,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 163709,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1106395,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1270104,
                            "linear_total": 7077888,
                            "nnz": 1280088,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 158018,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1044282,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1202300,
                            "linear_total": 7077888,
                            "nnz": 1212284,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 125746,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1010449,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1136195,
                            "linear_total": 7077888,
                            "nnz": 1146179,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 110023,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 861094,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 971117,
                            "linear_total": 7077888,
                            "nnz": 981101,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 113086,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 632989,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 746075,
                            "linear_total": 7077888,
                            "nnz": 756059,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 81879,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 407092,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 488971,
                            "linear_total": 7077888,
                            "nnz": 498955,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 77365,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 173330,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 250695,
                            "linear_total": 7077888,
                            "nnz": 260679,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 9959406,
                    "linear_sparsity": 88.27403739646628,
                    "linear_total": 84934656,
                    "nnz": 33917936,
                    "pruned_heads": {
                        "0": [
                            9
                        ],
                        "1": [
                            0,
                            8
                        ],
                        "10": [
                            8,
                            1,
                            4
                        ],
                        "11": [
                            8,
                            11,
                            5,
                            7
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [],
                        "5": [
                            1,
                            6,
                            7
                        ],
                        "6": [
                            2,
                            3
                        ],
                        "7": [
                            1,
                            11,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            4,
                            5
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 68.85210429971255
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.64967103979136,
            "fill_rate": 0.11725962603533713,
            "speedup": 1.170038217254783
        },
        {
            "annotate": "11",
            "cat_fun_name": "is_improved_mvmt_pruning",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.8391674550615,
                    "f1": 87.59923644792065
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test3/hp_od-output__squad_test3_es-steps_nte20_ls250_est5000_rn-output__squad_test3_dpm-sigmoied_threshold_apme-sigmoied_threshold_abr1_abc1_it0_fw10_r-l1_rfl75_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 75
                },
                "speed": {
                    "cuda_eval_elapsed_time": 33.06226232147217,
                    "eval_elapsed_time": 40.42444095481187
                },
                "speedup": 1.1673246261888772,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 56086,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1044542,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1100628,
                            "linear_total": 7077888,
                            "nnz": 1110612,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 115328,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1096450,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1211778,
                            "linear_total": 7077888,
                            "nnz": 1221762,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 50374,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 120861,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 171235,
                            "linear_total": 7077888,
                            "nnz": 181219,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 28038,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 93754,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 121792,
                            "linear_total": 7077888,
                            "nnz": 131776,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 125881,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1127188,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1253069,
                            "linear_total": 7077888,
                            "nnz": 1263053,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 161525,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1096986,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1258511,
                            "linear_total": 7077888,
                            "nnz": 1268495,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 155911,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1035794,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1191705,
                            "linear_total": 7077888,
                            "nnz": 1201689,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 123921,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1001507,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1125428,
                            "linear_total": 7077888,
                            "nnz": 1135412,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 108430,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 853489,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 961919,
                            "linear_total": 7077888,
                            "nnz": 971903,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 111505,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 627123,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 738628,
                            "linear_total": 7077888,
                            "nnz": 748612,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 80805,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 403383,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 484188,
                            "linear_total": 7077888,
                            "nnz": 494172,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 76456,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 171492,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 247948,
                            "linear_total": 7077888,
                            "nnz": 257932,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 9866829,
                    "linear_sparsity": 88.38303530657733,
                    "linear_total": 84934656,
                    "nnz": 33825359,
                    "pruned_heads": {
                        "0": [
                            9
                        ],
                        "1": [
                            0,
                            8
                        ],
                        "10": [
                            8,
                            1,
                            4
                        ],
                        "11": [
                            8,
                            11,
                            5,
                            7
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [],
                        "5": [
                            1,
                            6,
                            7
                        ],
                        "6": [
                            2,
                            3
                        ],
                        "7": [
                            1,
                            11,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            4,
                            5
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 68.93712063856779
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test3",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "output_dir": "output/squad_test3",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test3",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.59923644792065,
            "fill_rate": 0.11616964693422671,
            "speedup": 1.1673246261888772
        },
        {
            "annotate": "11",
            "cat_fun_name": "is_improved_mvmt_pruning",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 81.93945127719962,
                    "f1": 88.66959543954316
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_8_mvp_lt/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test_8_mvp_lt___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test_8_mvp_lt___dpm-si--3dbebc278974335e/checkpoint-100000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 80
                },
                "speed": {
                    "cuda_eval_elapsed_time": 35.22903040313721,
                    "eval_elapsed_time": 42.20514475926757
                },
                "speedup": 1.095528107464865,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 159799,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1019941,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1179740,
                            "linear_total": 7077888,
                            "nnz": 1188733,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 225256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1060626,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1285882,
                            "linear_total": 7077888,
                            "nnz": 1294828,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 95768,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 108189,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 203957,
                            "linear_total": 7077888,
                            "nnz": 211096,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 54825,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 81043,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 135868,
                            "linear_total": 7077888,
                            "nnz": 142072,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 297063,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1050962,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1348025,
                            "linear_total": 7077888,
                            "nnz": 1357400,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 344394,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 999457,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1343851,
                            "linear_total": 7077888,
                            "nnz": 1353282,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 360638,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 868456,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1229094,
                            "linear_total": 7077888,
                            "nnz": 1238822,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 279458,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 803587,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1083045,
                            "linear_total": 7077888,
                            "nnz": 1092686,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 255911,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 634016,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 889927,
                            "linear_total": 7077888,
                            "nnz": 899603,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 204977,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 440993,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 645970,
                            "linear_total": 7077888,
                            "nnz": 655018,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 179070,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 265135,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 444205,
                            "linear_total": 7077888,
                            "nnz": 453337,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 124400,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 124282,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 248682,
                            "linear_total": 7077888,
                            "nnz": 256304,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 10038246,
                    "linear_sparsity": 88.18121309633608,
                    "linear_total": 84934656,
                    "nnz": 33981903,
                    "pruned_heads": {
                        "0": [],
                        "1": [],
                        "10": [
                            1,
                            2,
                            4,
                            5
                        ],
                        "11": [
                            8,
                            5
                        ],
                        "2": [],
                        "3": [],
                        "4": [],
                        "5": [],
                        "6": [
                            3
                        ],
                        "7": [],
                        "8": [
                            0
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 68.79336141381702
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_8_mvp_lt/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_8_mvp_lt/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_8_mvp_lt/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.66959543954316,
            "fill_rate": 0.11818786903663925,
            "speedup": 1.095528107464865
        },
        {
            "annotate": "11",
            "cat_fun_name": "is_improved_mvmt_pruning",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 81.75023651844845,
                    "f1": 88.66465208237972
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_8_mvp_lt/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test_8_mvp_lt___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test_8_mvp_lt___dpm-si--3dbebc278974335e/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 80
                },
                "speed": {
                    "cuda_eval_elapsed_time": 35.25411331939697,
                    "eval_elapsed_time": 42.25662731193006
                },
                "speedup": 1.0947486511917544,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 152512,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 983073,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1135585,
                            "linear_total": 7077888,
                            "nnz": 1144544,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 216347,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1024604,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1240951,
                            "linear_total": 7077888,
                            "nnz": 1249883,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 92277,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 104692,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 196969,
                            "linear_total": 7077888,
                            "nnz": 204060,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 53199,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 79081,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 132280,
                            "linear_total": 7077888,
                            "nnz": 138490,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 285426,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1014981,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1300407,
                            "linear_total": 7077888,
                            "nnz": 1309761,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 330893,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 964681,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1295574,
                            "linear_total": 7077888,
                            "nnz": 1304991,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 345899,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 836618,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1182517,
                            "linear_total": 7077888,
                            "nnz": 1192226,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 265858,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 773177,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1039035,
                            "linear_total": 7077888,
                            "nnz": 1048663,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 243070,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 608552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 851622,
                            "linear_total": 7077888,
                            "nnz": 861274,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 195594,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 422844,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 618438,
                            "linear_total": 7077888,
                            "nnz": 627457,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 170492,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 254582,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 425074,
                            "linear_total": 7077888,
                            "nnz": 434164,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 119453,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 119226,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 238679,
                            "linear_total": 7077888,
                            "nnz": 246241,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 9657131,
                    "linear_sparsity": 88.62992863596222,
                    "linear_total": 84934656,
                    "nnz": 33600476,
                    "pruned_heads": {
                        "0": [],
                        "1": [],
                        "10": [
                            1,
                            2,
                            4,
                            5
                        ],
                        "11": [
                            8,
                            5
                        ],
                        "2": [],
                        "3": [],
                        "4": [],
                        "5": [],
                        "6": [
                            3
                        ],
                        "7": [
                            7
                        ],
                        "8": [
                            0
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 69.1436376928121
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_8_mvp_lt/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_8_mvp_lt/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_8_mvp_lt/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.66465208237972,
            "fill_rate": 0.11370071364037782,
            "speedup": 1.0947486511917544
        },
        {
            "annotate": "19",
            "cat_fun_name": "is_improved_mvmt_pruning",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 83.17880794701986,
                    "f1": 89.78322305629628
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_8_mvp_lt/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test_8_mvp_lt___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test_8_mvp_lt___dpm-si--45bf1e1da1b7299c/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 40
                },
                "speed": {
                    "cuda_eval_elapsed_time": 37.61116293334961,
                    "eval_elapsed_time": 44.6719565698877
                },
                "speedup": 1.026141974757969,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 270490,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1480949,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1751439,
                            "linear_total": 7077888,
                            "nnz": 1760894,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 335933,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1534020,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1869953,
                            "linear_total": 7077888,
                            "nnz": 1879366,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 166873,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 247010,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 413883,
                            "linear_total": 7077888,
                            "nnz": 422281,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 93795,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 153368,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 247163,
                            "linear_total": 7077888,
                            "nnz": 254059,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 443918,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1543955,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1987873,
                            "linear_total": 7077888,
                            "nnz": 1997593,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 509017,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1515526,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2024543,
                            "linear_total": 7077888,
                            "nnz": 2034242,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 560867,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1397000,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1957867,
                            "linear_total": 7077888,
                            "nnz": 1967760,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 481964,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1339614,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1821578,
                            "linear_total": 7077888,
                            "nnz": 1831430,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 465111,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1115521,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1580632,
                            "linear_total": 7077888,
                            "nnz": 1590485,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 349655,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 836653,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1186308,
                            "linear_total": 7077888,
                            "nnz": 1195564,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 347514,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 546212,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 893726,
                            "linear_total": 7077888,
                            "nnz": 903465,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 229122,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 295840,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 524962,
                            "linear_total": 7077888,
                            "nnz": 533702,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 16259927,
                    "linear_sparsity": 80.85595707834503,
                    "linear_total": 84934656,
                    "nnz": 40209563,
                    "pruned_heads": {
                        "0": [],
                        "1": [],
                        "10": [
                            1,
                            4
                        ],
                        "11": [
                            5
                        ],
                        "2": [],
                        "3": [],
                        "4": [],
                        "5": [],
                        "6": [],
                        "7": [],
                        "8": [],
                        "9": [
                            4
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 63.07430751452161
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_8_mvp_lt/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_8_mvp_lt/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_8_mvp_lt/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 89.78322305629628,
            "fill_rate": 0.1914404292165497,
            "speedup": 1.026141974757969
        },
        {
            "annotate": "30",
            "cat_fun_name": "is_improved_mvmt_pruning",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 83.87890255439925,
                    "f1": 90.24019516114679
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_8_mvp_lt/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test_8_mvp_lt___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test_8_mvp_lt___dpm-si--5742d4278f871b20/checkpoint-105000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20
                },
                "speed": {
                    "cuda_eval_elapsed_time": 39.22058818817139,
                    "eval_elapsed_time": 46.20080855116248
                },
                "speedup": 0.9840340185670864,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 506421,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2119553,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2625974,
                            "linear_total": 7077888,
                            "nnz": 2635887,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 576671,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2157695,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2734366,
                            "linear_total": 7077888,
                            "nnz": 2744311,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 325235,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 597278,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 922513,
                            "linear_total": 7077888,
                            "nnz": 931676,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 195895,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 315779,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 511674,
                            "linear_total": 7077888,
                            "nnz": 519723,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 722482,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2188133,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2910615,
                            "linear_total": 7077888,
                            "nnz": 2920599,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 801394,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2185014,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2986408,
                            "linear_total": 7077888,
                            "nnz": 2996347,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 869061,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2071237,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2940298,
                            "linear_total": 7077888,
                            "nnz": 2950272,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 816637,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2032020,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2848657,
                            "linear_total": 7077888,
                            "nnz": 2858639,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 812906,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1802564,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2615470,
                            "linear_total": 7077888,
                            "nnz": 2625454,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 601681,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1468047,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2069728,
                            "linear_total": 7077888,
                            "nnz": 2079319,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 674649,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1053978,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1728627,
                            "linear_total": 7077888,
                            "nnz": 1738609,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 449718,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 674197,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1123915,
                            "linear_total": 7077888,
                            "nnz": 1133393,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 26018245,
                    "linear_sparsity": 69.36675059942552,
                    "linear_total": 84934656,
                    "nnz": 49972951,
                    "pruned_heads": {
                        "0": [],
                        "1": [],
                        "10": [],
                        "11": [],
                        "2": [],
                        "3": [],
                        "4": [],
                        "5": [],
                        "6": [],
                        "7": [],
                        "8": [],
                        "9": []
                    },
                    "total": 108893186,
                    "total_sparsity": 54.10828460836843
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_8_mvp_lt/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_8_mvp_lt/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_8_mvp_lt/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 90.24019516114679,
            "fill_rate": 0.3063324940057448,
            "speedup": 0.9840340185670864
        },
        {
            "annotate": "30",
            "cat_fun_name": "is_improved_mvmt_pruning",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 83.83159886471145,
                    "f1": 90.15245565366504
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_8_mvp_lt/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test_8_mvp_lt___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test_8_mvp_lt___dpm-si--5742d4278f871b20/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20
                },
                "speed": {
                    "cuda_eval_elapsed_time": 39.27320726776123,
                    "eval_elapsed_time": 46.266036483459175
                },
                "speedup": 0.9827155888297577,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 500124,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2106019,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2606143,
                            "linear_total": 7077888,
                            "nnz": 2616044,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 569798,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2144520,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2714318,
                            "linear_total": 7077888,
                            "nnz": 2724261,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 322107,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 591499,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 913606,
                            "linear_total": 7077888,
                            "nnz": 922764,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 193582,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 313141,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 506723,
                            "linear_total": 7077888,
                            "nnz": 514745,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 714815,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2175090,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2889905,
                            "linear_total": 7077888,
                            "nnz": 2899889,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 794014,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2172123,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2966137,
                            "linear_total": 7077888,
                            "nnz": 2976073,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 861683,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2058780,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2920463,
                            "linear_total": 7077888,
                            "nnz": 2930437,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 808888,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 2019153,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2828041,
                            "linear_total": 7077888,
                            "nnz": 2838023,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 805274,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1790540,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2595814,
                            "linear_total": 7077888,
                            "nnz": 2605798,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 596126,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1457138,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2053264,
                            "linear_total": 7077888,
                            "nnz": 2062853,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 667516,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1045380,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1712896,
                            "linear_total": 7077888,
                            "nnz": 1722878,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 445094,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 667985,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1113079,
                            "linear_total": 7077888,
                            "nnz": 1122551,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 25820389,
                    "linear_sparsity": 69.59970144577969,
                    "linear_total": 84934656,
                    "nnz": 49775038,
                    "pruned_heads": {
                        "0": [],
                        "1": [],
                        "10": [],
                        "11": [],
                        "2": [],
                        "3": [],
                        "4": [],
                        "5": [],
                        "6": [],
                        "7": [],
                        "8": [],
                        "9": []
                    },
                    "total": 108893186,
                    "total_sparsity": 54.290034272667896
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_8_mvp_lt/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_8_mvp_lt/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_8_mvp_lt/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 90.15245565366504,
            "fill_rate": 0.3040029855422032,
            "speedup": 0.9827155888297577
        },
        {
            "annotate": "6",
            "cat_fun_name": "is_improved_mvmt_pruning",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.09460737937559,
                    "f1": 87.45347230995543
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_8_mvp_lt/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test_8_mvp_lt___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test_8_mvp_lt___dpm-si--7ebf7572d80fe282/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 160
                },
                "speed": {
                    "cuda_eval_elapsed_time": 30.61294924926758,
                    "eval_elapsed_time": 37.62639235612005
                },
                "speedup": 1.2607211638158147,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 85933,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 599767,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 685700,
                            "linear_total": 7077888,
                            "nnz": 694282,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 146970,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 633851,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 780821,
                            "linear_total": 7077888,
                            "nnz": 789490,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 53976,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 51816,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 105792,
                            "linear_total": 7077888,
                            "nnz": 111956,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 35246,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 44570,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 79816,
                            "linear_total": 7077888,
                            "nnz": 85561,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 185080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 607798,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 792878,
                            "linear_total": 7077888,
                            "nnz": 802001,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 215120,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 555649,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 770769,
                            "linear_total": 7077888,
                            "nnz": 779934,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 203907,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 453085,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 656992,
                            "linear_total": 7077888,
                            "nnz": 666424,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 135767,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 395091,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 530858,
                            "linear_total": 7077888,
                            "nnz": 539768,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 123379,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 299238,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 422617,
                            "linear_total": 7077888,
                            "nnz": 431706,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 111774,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 196632,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 308406,
                            "linear_total": 7077888,
                            "nnz": 316679,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 84158,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 114682,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 198840,
                            "linear_total": 7077888,
                            "nnz": 206795,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 68642,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 52118,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 120760,
                            "linear_total": 7077888,
                            "nnz": 126954,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 5454249,
                    "linear_sparsity": 93.5782997696488,
                    "linear_total": 84934656,
                    "nnz": 29390272,
                    "pruned_heads": {
                        "0": [
                            9
                        ],
                        "1": [],
                        "10": [
                            1,
                            2,
                            4,
                            5
                        ],
                        "11": [
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [],
                        "5": [
                            1,
                            6,
                            7
                        ],
                        "6": [
                            2,
                            3
                        ],
                        "7": [
                            1,
                            11,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 73.00999899112144
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_8_mvp_lt/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_8_mvp_lt/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_8_mvp_lt/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 87.45347230995543,
            "fill_rate": 0.06421700230351202,
            "speedup": 1.2607211638158147
        },
        {
            "annotate": "8",
            "cat_fun_name": "is_improved_mvmt_pruning",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 80.93661305581836,
                    "f1": 88.11360890595924
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_8_mvp_lt/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test_8_mvp_lt___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test_8_mvp_lt___dpm-si--7fe43555f854fbb6/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 120
                },
                "speed": {
                    "cuda_eval_elapsed_time": 33.17768461608887,
                    "eval_elapsed_time": 40.18306041043252
                },
                "speedup": 1.1632636047982534,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 108638,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 743508,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 852146,
                            "linear_total": 7077888,
                            "nnz": 860873,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 171340,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 780119,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 951459,
                            "linear_total": 7077888,
                            "nnz": 960245,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 67316,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 69353,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 136669,
                            "linear_total": 7077888,
                            "nnz": 143162,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 41497,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 55855,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 97352,
                            "linear_total": 7077888,
                            "nnz": 103286,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 221074,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 756625,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 977699,
                            "linear_total": 7077888,
                            "nnz": 986901,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 258229,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 705908,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 964137,
                            "linear_total": 7077888,
                            "nnz": 973427,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 255136,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 593338,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 848474,
                            "linear_total": 7077888,
                            "nnz": 858037,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 179994,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 531061,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 711055,
                            "linear_total": 7077888,
                            "nnz": 720395,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 165167,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 406391,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 571558,
                            "linear_total": 7077888,
                            "nnz": 580963,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 139907,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 272514,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 412421,
                            "linear_total": 7077888,
                            "nnz": 421032,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 113253,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 163778,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 277031,
                            "linear_total": 7077888,
                            "nnz": 285536,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 84915,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 71190,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 156105,
                            "linear_total": 7077888,
                            "nnz": 162775,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 6956106,
                    "linear_sparsity": 91.81004983407479,
                    "linear_total": 84934656,
                    "nnz": 30895354,
                    "pruned_heads": {
                        "0": [
                            9
                        ],
                        "1": [],
                        "10": [
                            1,
                            2,
                            4
                        ],
                        "11": [
                            8,
                            11,
                            5,
                            7
                        ],
                        "2": [
                            8
                        ],
                        "3": [
                            2,
                            4
                        ],
                        "4": [],
                        "5": [
                            1
                        ],
                        "6": [
                            2,
                            3
                        ],
                        "7": [
                            1,
                            7
                        ],
                        "8": [
                            0
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 71.62783537254572
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_8_mvp_lt/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_8_mvp_lt/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_8_mvp_lt/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 88.11360890595924,
            "fill_rate": 0.08189950165925208,
            "speedup": 1.1632636047982534
        },
        {
            "annotate": "4",
            "cat_fun_name": "is_improved_mvmt_pruning",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.42005676442763,
                    "f1": 86.50729252303553
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test_8_mvp_lt/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test_8_mvp_lt___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test_8_mvp_lt___dpm-sig--51ab88e6fe9e0cb/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 240
                },
                "speed": {
                    "cuda_eval_elapsed_time": 28.90965769958496,
                    "eval_elapsed_time": 35.917956955730915
                },
                "speedup": 1.3349999991845345,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 59018,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 429594,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 488612,
                            "linear_total": 7077888,
                            "nnz": 496889,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 117573,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 452265,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 569838,
                            "linear_total": 7077888,
                            "nnz": 578325,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 41874,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 36903,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 78777,
                            "linear_total": 7077888,
                            "nnz": 84672,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 27872,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 32456,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 60328,
                            "linear_total": 7077888,
                            "nnz": 65863,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 144344,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 429183,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 573527,
                            "linear_total": 7077888,
                            "nnz": 582495,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 168864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 379065,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 547929,
                            "linear_total": 7077888,
                            "nnz": 556868,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 140816,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 300856,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 441672,
                            "linear_total": 7077888,
                            "nnz": 450547,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 93231,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 250434,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 343665,
                            "linear_total": 7077888,
                            "nnz": 352041,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 81345,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 193259,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 274604,
                            "linear_total": 7077888,
                            "nnz": 282887,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 83004,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 127052,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 210056,
                            "linear_total": 7077888,
                            "nnz": 217609,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 56265,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 72037,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 128302,
                            "linear_total": 7077888,
                            "nnz": 135460,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 50559,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 35373,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 85932,
                            "linear_total": 7077888,
                            "nnz": 91704,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 3803242,
                    "linear_sparsity": 95.52215529076847,
                    "linear_total": 84934656,
                    "nnz": 27734082,
                    "pruned_heads": {
                        "0": [
                            9
                        ],
                        "1": [
                            8,
                            5
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7
                        ],
                        "11": [
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [],
                        "5": [
                            1,
                            6,
                            7
                        ],
                        "6": [
                            2,
                            3
                        ],
                        "7": [
                            1,
                            11,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            4
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 74.53092978655249
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_8_mvp_lt/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test_8_mvp_lt/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test_8_mvp_lt/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.50729252303553,
            "fill_rate": 0.04477844709231538,
            "speedup": 1.3349999991845345
        }
    ],
    "new method, attention pruned with rows": [
        {
            "annotate": "",
            "cat_fun_name": "is_attention_rows",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 79.15799432355723,
                    "f1": 86.94169166073364
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test/hp_es-steps_nte20_ls250_est5000_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_abr1_it0_fw10_r-l1_rfl5_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-110000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 768,
                    "attention_block_rows": 1,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 5
                },
                "speed": {
                    "cuda_eval_elapsed_time": 22.747020225524903,
                    "eval_elapsed_time": 29.958857133984566
                },
                "speedup": 1.6966790648941144,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 427776,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1617408,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2045184,
                            "linear_total": 7077888,
                            "nnz": 2055168,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 394752,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1708032,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2102784,
                            "linear_total": 7077888,
                            "nnz": 2112768,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 182784,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 245760,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 428544,
                            "linear_total": 7077888,
                            "nnz": 438528,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 112128,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 626688,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 738816,
                            "linear_total": 7077888,
                            "nnz": 748800,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 469248,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1955328,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2424576,
                            "linear_total": 7077888,
                            "nnz": 2434560,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 579840,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1923072,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2502912,
                            "linear_total": 7077888,
                            "nnz": 2512896,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 539904,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1837056,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2376960,
                            "linear_total": 7077888,
                            "nnz": 2386944,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 424704,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1777152,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2201856,
                            "linear_total": 7077888,
                            "nnz": 2211840,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 439296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1468416,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1907712,
                            "linear_total": 7077888,
                            "nnz": 1917696,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 428544,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1152000,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1580544,
                            "linear_total": 7077888,
                            "nnz": 1590528,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 397824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 697344,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1095168,
                            "linear_total": 7077888,
                            "nnz": 1105152,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 235776,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 291840,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 527616,
                            "linear_total": 7077888,
                            "nnz": 537600,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 19932672,
                    "linear_sparsity": 76.53175636574075,
                    "linear_total": 84934656,
                    "nnz": 43891202,
                    "pruned_heads": {
                        "0": [
                            2
                        ],
                        "1": [
                            8,
                            2,
                            6
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7
                        ],
                        "11": [
                            0,
                            5,
                            6,
                            7,
                            8,
                            11
                        ],
                        "2": [
                            8,
                            1,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [],
                        "5": [
                            1,
                            6
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            7
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 59.6933438975695
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "output_dir": "output/squad_test",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.94169166073364,
            "fill_rate": 0.23468243634259256,
            "speedup": 1.6966790648941144
        }
    ],
    "new method, longer final warmup (15 instead of 10)": [
        {
            "annotate": "",
            "cat_fun_name": "is_longer_final_warmup",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.88363292336803,
                    "f1": 86.63235572290178
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test2/hp_od-output__squad_test2_es-steps_nte30_ls250_est5000_rn-output__squad_test2_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_abr32_abc32_it0_fw15_r-l1_rfl10_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-130000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 15,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 15.435867366790772,
                    "eval_elapsed_time": 22.581966675817966
                },
                "speedup": 2.500306078581909,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 488448,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 562176,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1050624,
                            "linear_total": 7077888,
                            "nnz": 1060608,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 512512,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 870912,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1383424,
                            "linear_total": 7077888,
                            "nnz": 1393408,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 367360,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 116736,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 484096,
                            "linear_total": 7077888,
                            "nnz": 494080,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 225024,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 271872,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 496896,
                            "linear_total": 7077888,
                            "nnz": 506880,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 628224,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 999936,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1628160,
                            "linear_total": 7077888,
                            "nnz": 1638144,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 937216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1061376,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1998592,
                            "linear_total": 7077888,
                            "nnz": 2008576,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 821760,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1118208,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1939968,
                            "linear_total": 7077888,
                            "nnz": 1949952,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 648448,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1061376,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1709824,
                            "linear_total": 7077888,
                            "nnz": 1719808,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 641536,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 763392,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1404928,
                            "linear_total": 7077888,
                            "nnz": 1414912,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 755712,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 605184,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1360896,
                            "linear_total": 7077888,
                            "nnz": 1370880,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 467712,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 350208,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 817920,
                            "linear_total": 7077888,
                            "nnz": 827904,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 403200,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 141312,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 544512,
                            "linear_total": 7077888,
                            "nnz": 554496,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 14819840,
                    "linear_sparsity": 82.5514805169753,
                    "linear_total": 84934656,
                    "nnz": 38778370,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 64.38861656596218
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test2",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 30,
                    "output_dir": "output/squad_test2",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test2",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                },
                "unopt_eval_metrics": {
                    "exact_match": 78.9593188268685,
                    "f1": 86.73375735833712
                }
            },
            "f1": 86.63235572290178,
            "fill_rate": 0.17448519483024694,
            "speedup": 2.500306078581909
        },
        {
            "annotate": "",
            "cat_fun_name": "is_longer_final_warmup",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "exact_match": 78.96877956480606,
                    "f1": 86.71968503618079
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_test2/hp_od-output__squad_test2_es-steps_nte30_ls250_est5000_rn-output__squad_test2_dpm-sigmoied_threshold:1d_alt_apme-sigmoied_threshold_abr32_abc32_it0_fw15_r-l1_rfl10_dtnop-csarron__bert-base-uncased-squad-v1/checkpoint-165000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_pruning_method": "sigmoied_threshold",
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "csarron/bert-base-uncased-squad-v1",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_threshold": 0.1,
                    "final_warmup": 15,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 15.359982524871826,
                    "eval_elapsed_time": 22.516427854076028
                },
                "speedup": 2.512658653281453,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 468480,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 542208,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1010688,
                            "linear_total": 7077888,
                            "nnz": 1020672,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 518912,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 852480,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1371392,
                            "linear_total": 7077888,
                            "nnz": 1381376,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 345344,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 110592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 455936,
                            "linear_total": 7077888,
                            "nnz": 465920,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 212992,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 261120,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 474112,
                            "linear_total": 7077888,
                            "nnz": 484096,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 608768,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 981504,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1590272,
                            "linear_total": 7077888,
                            "nnz": 1600256,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 869888,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1026048,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1895936,
                            "linear_total": 7077888,
                            "nnz": 1905920,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 775936,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1093632,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1869568,
                            "linear_total": 7077888,
                            "nnz": 1879552,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 618752,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1044480,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1663232,
                            "linear_total": 7077888,
                            "nnz": 1673216,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 629248,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 754176,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1383424,
                            "linear_total": 7077888,
                            "nnz": 1393408,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 707584,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 588288,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1295872,
                            "linear_total": 7077888,
                            "nnz": 1305856,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 463104,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 345600,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 808704,
                            "linear_total": 7077888,
                            "nnz": 818688,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 376064,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 139776,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 515840,
                            "linear_total": 7077888,
                            "nnz": 525824,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 14334976,
                    "linear_sparsity": 83.1223476080247,
                    "linear_total": 84934656,
                    "nnz": 38293506,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            2,
                            4,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            6,
                            11
                        ],
                        "5": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            0,
                            2,
                            3,
                            4,
                            7,
                            10
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            6,
                            8
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9,
                            10
                        ]
                    },
                    "total": 108893186,
                    "total_sparsity": 64.83388225963009
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/squad_test2",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 30,
                    "output_dir": "output/squad_test2",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 8,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/squad_test2",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 5,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 86.71968503618079,
            "fill_rate": 0.16877652391975306,
            "speedup": 2.512658653281453
        }
    ]
}