{
    "Block/struct method, bs= 32x32, v=1, s=b": [
        {
            "annotate": "33",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "transformers_version": "4.4.2",
                    "type_vocab_size": 2,
                    "use_cache": true,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "HasAns_exact": 70.49595141700405,
                    "HasAns_f1": 77.05313447818509,
                    "HasAns_total": 5928,
                    "NoAns_exact": 74.01177460050462,
                    "NoAns_f1": 74.01177460050462,
                    "NoAns_total": 5945,
                    "best_exact": 72.25638002189842,
                    "best_exact_thresh": 0.0,
                    "best_f1": 75.53027719924893,
                    "best_f1_thresh": 0.0,
                    "exact": 72.25638002189842,
                    "f1": 75.53027719924886,
                    "total": 11873
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--1d4e4bc55a9a2bf2/checkpoint-135000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": false,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": false,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "linear_min_parameters": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 5.0,
                    "rewind_model_name_or_path": null
                },
                "speed": {
                    "cuda_eval_elapsed_time": 26.228707473754884,
                    "eval_elapsed_time": 30.511660314979963
                },
                "speedup": 1.6825629567633156,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1516544,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1310208,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2826752,
                            "linear_total": 7077888,
                            "nnz": 2834101,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 1188864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1448448,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2637312,
                            "linear_total": 7077888,
                            "nnz": 2644527,
                            "total": 7087488
                        },
                        "10": {
                            "linear_attention_nnz": 746496,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 253440,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 999936,
                            "linear_total": 7077888,
                            "nnz": 1005765,
                            "total": 7086912
                        },
                        "11": {
                            "linear_attention_nnz": 486400,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 397824,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 884224,
                            "linear_total": 7077888,
                            "nnz": 890051,
                            "total": 7086912
                        },
                        "2": {
                            "linear_attention_nnz": 1540096,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1523712,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3063808,
                            "linear_total": 7077888,
                            "nnz": 3071200,
                            "total": 7087488
                        },
                        "3": {
                            "linear_attention_nnz": 1534976,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1637376,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3172352,
                            "linear_total": 7077888,
                            "nnz": 3179754,
                            "total": 7087296
                        },
                        "4": {
                            "linear_attention_nnz": 1934336,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1545216,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3479552,
                            "linear_total": 7077888,
                            "nnz": 3487470,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1677312,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1449984,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3127296,
                            "linear_total": 7077888,
                            "nnz": 3134768,
                            "total": 7087488
                        },
                        "6": {
                            "linear_attention_nnz": 1732608,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1130496,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2863104,
                            "linear_total": 7077888,
                            "nnz": 2870560,
                            "total": 7087680
                        },
                        "7": {
                            "linear_attention_nnz": 1282048,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 824832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2106880,
                            "linear_total": 7077888,
                            "nnz": 2113561,
                            "total": 7087104
                        },
                        "8": {
                            "linear_attention_nnz": 1499136,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 463872,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1963008,
                            "linear_total": 7077888,
                            "nnz": 1969838,
                            "total": 7087680
                        },
                        "9": {
                            "linear_attention_nnz": 839680,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 213504,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1053184,
                            "linear_total": 7077888,
                            "nnz": 1059083,
                            "total": 7086912
                        }
                    },
                    "linear_nnz": 28177408,
                    "linear_sparsity": 66.82460455246914,
                    "linear_total": 84934656,
                    "nnz": 52099400,
                    "pruned_heads": {
                        "0": [],
                        "1": [
                            0,
                            5
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            8
                        ],
                        "11": [
                            3,
                            5,
                            7,
                            8,
                            11
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            3
                        ],
                        "7": [
                            3,
                            11,
                            6,
                            7
                        ],
                        "8": [
                            0
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "total": 108887426,
                    "total_sparsity": 52.15296943468937
                },
                "training_args": {
                    "_n_gpu": 1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "length_column_name": "length",
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "mp_parameters": "",
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": [
                        "tensorboard",
                        "wandb"
                    ],
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": [],
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 75.53027719924886,
            "fill_rate": 0.33175395447530864,
            "speedup": 1.6825629567633156
        },
        {
            "annotate": "32",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "transformers_version": "4.4.2",
                    "type_vocab_size": 2,
                    "use_cache": true,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "HasAns_exact": 71.5080971659919,
                    "HasAns_f1": 78.18165134334741,
                    "HasAns_total": 5928,
                    "NoAns_exact": 72.80067283431455,
                    "NoAns_f1": 72.80067283431455,
                    "NoAns_total": 5945,
                    "best_exact": 72.15531036806199,
                    "best_exact_thresh": 0.0,
                    "best_f1": 75.48730979224834,
                    "best_f1_thresh": 0.0,
                    "exact": 72.15531036806199,
                    "f1": 75.48730979224833,
                    "total": 11873
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--1d4e4bc55a9a2bf2/checkpoint-150000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": false,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": false,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "linear_min_parameters": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 5.0,
                    "rewind_model_name_or_path": null
                },
                "speed": {
                    "cuda_eval_elapsed_time": 26.09268295288086,
                    "eval_elapsed_time": 30.314951282925904
                },
                "speedup": 1.691334374422719,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1448960,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1261056,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2710016,
                            "linear_total": 7077888,
                            "nnz": 2717141,
                            "total": 7087680
                        },
                        "1": {
                            "linear_attention_nnz": 1113088,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1402368,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2515456,
                            "linear_total": 7077888,
                            "nnz": 2522545,
                            "total": 7087488
                        },
                        "10": {
                            "linear_attention_nnz": 733184,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 245760,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 978944,
                            "linear_total": 7077888,
                            "nnz": 984768,
                            "total": 7086912
                        },
                        "11": {
                            "linear_attention_nnz": 475136,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 388608,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 863744,
                            "linear_total": 7077888,
                            "nnz": 869565,
                            "total": 7086912
                        },
                        "2": {
                            "linear_attention_nnz": 1537024,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1482240,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3019264,
                            "linear_total": 7077888,
                            "nnz": 3026597,
                            "total": 7087488
                        },
                        "3": {
                            "linear_attention_nnz": 1533952,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1598976,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3132928,
                            "linear_total": 7077888,
                            "nnz": 3140305,
                            "total": 7087296
                        },
                        "4": {
                            "linear_attention_nnz": 1928192,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1525248,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3453440,
                            "linear_total": 7077888,
                            "nnz": 3461345,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1634304,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1416192,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3050496,
                            "linear_total": 7077888,
                            "nnz": 3057946,
                            "total": 7087488
                        },
                        "6": {
                            "linear_attention_nnz": 1678336,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1110528,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2788864,
                            "linear_total": 7077888,
                            "nnz": 2796275,
                            "total": 7087680
                        },
                        "7": {
                            "linear_attention_nnz": 1271808,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 804864,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2076672,
                            "linear_total": 7077888,
                            "nnz": 2083340,
                            "total": 7087104
                        },
                        "8": {
                            "linear_attention_nnz": 1433600,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 456192,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1889792,
                            "linear_total": 7077888,
                            "nnz": 1896585,
                            "total": 7087680
                        },
                        "9": {
                            "linear_attention_nnz": 838656,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 213504,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1052160,
                            "linear_total": 7077888,
                            "nnz": 1058059,
                            "total": 7086912
                        }
                    },
                    "linear_nnz": 27531776,
                    "linear_sparsity": 67.58475597993827,
                    "linear_total": 84934656,
                    "nnz": 51453193,
                    "pruned_heads": {
                        "0": [
                            4
                        ],
                        "1": [
                            0,
                            5
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            8
                        ],
                        "11": [
                            3,
                            5,
                            7,
                            8,
                            11
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            3
                        ],
                        "7": [
                            3,
                            11,
                            6,
                            7
                        ],
                        "8": [
                            0
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "total": 108887234,
                    "total_sparsity": 52.74634949400956
                },
                "training_args": {
                    "_n_gpu": 1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "length_column_name": "length",
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "mp_parameters": "",
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": [
                        "tensorboard",
                        "wandb"
                    ],
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": [],
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 75.48730979224833,
            "fill_rate": 0.3241524402006174,
            "speedup": 1.691334374422719
        },
        {
            "annotate": "32",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "transformers_version": "4.4.2",
                    "type_vocab_size": 2,
                    "use_cache": true,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "HasAns_exact": 71.9804318488529,
                    "HasAns_f1": 78.86810465582569,
                    "HasAns_total": 5928,
                    "NoAns_exact": 71.75777964676199,
                    "NoAns_f1": 71.75777964676199,
                    "NoAns_total": 5945,
                    "best_exact": 71.86894634885876,
                    "best_exact_thresh": 0.0,
                    "best_f1": 75.30785179817516,
                    "best_f1_thresh": 0.0,
                    "exact": 71.86894634885876,
                    "f1": 75.30785179817528,
                    "total": 11873
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--1d4e4bc55a9a2bf2/checkpoint-155000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": false,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": false,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "linear_min_parameters": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 5.0,
                    "rewind_model_name_or_path": null
                },
                "speed": {
                    "cuda_eval_elapsed_time": 26.10106954956055,
                    "eval_elapsed_time": 30.318406603997573
                },
                "speedup": 1.6907909277558364,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1435648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1256448,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2692096,
                            "linear_total": 7077888,
                            "nnz": 2699218,
                            "total": 7087680
                        },
                        "1": {
                            "linear_attention_nnz": 1107968,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1397760,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2505728,
                            "linear_total": 7077888,
                            "nnz": 2512814,
                            "total": 7087488
                        },
                        "10": {
                            "linear_attention_nnz": 747520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 244224,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 991744,
                            "linear_total": 7077888,
                            "nnz": 997567,
                            "total": 7086912
                        },
                        "11": {
                            "linear_attention_nnz": 503808,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 387072,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 890880,
                            "linear_total": 7077888,
                            "nnz": 896700,
                            "total": 7086912
                        },
                        "2": {
                            "linear_attention_nnz": 1502208,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1476096,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2978304,
                            "linear_total": 7077888,
                            "nnz": 2985633,
                            "total": 7087488
                        },
                        "3": {
                            "linear_attention_nnz": 1547264,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1592832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3140096,
                            "linear_total": 7077888,
                            "nnz": 3147469,
                            "total": 7087296
                        },
                        "4": {
                            "linear_attention_nnz": 1927168,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1525248,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3452416,
                            "linear_total": 7077888,
                            "nnz": 3460321,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1636352,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1413120,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3049472,
                            "linear_total": 7077888,
                            "nnz": 3056920,
                            "total": 7087488
                        },
                        "6": {
                            "linear_attention_nnz": 1682432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1107456,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2789888,
                            "linear_total": 7077888,
                            "nnz": 2797265,
                            "total": 7087680
                        },
                        "7": {
                            "linear_attention_nnz": 1293312,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 801792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2095104,
                            "linear_total": 7077888,
                            "nnz": 2101770,
                            "total": 7087104
                        },
                        "8": {
                            "linear_attention_nnz": 1396736,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 456192,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1852928,
                            "linear_total": 7077888,
                            "nnz": 1859721,
                            "total": 7087680
                        },
                        "9": {
                            "linear_attention_nnz": 815104,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 211968,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1027072,
                            "linear_total": 7077888,
                            "nnz": 1032970,
                            "total": 7086912
                        }
                    },
                    "linear_nnz": 27465728,
                    "linear_sparsity": 67.66251929012346,
                    "linear_total": 84934656,
                    "nnz": 51387090,
                    "pruned_heads": {
                        "0": [
                            4
                        ],
                        "1": [
                            0,
                            5
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            8
                        ],
                        "11": [
                            3,
                            5,
                            7,
                            8,
                            11
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            3
                        ],
                        "7": [
                            3,
                            11,
                            6,
                            7
                        ],
                        "8": [
                            0
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "total": 108887234,
                    "total_sparsity": 52.8070572533783
                },
                "training_args": {
                    "_n_gpu": 1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "length_column_name": "length",
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "mp_parameters": "",
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": [
                        "tensorboard",
                        "wandb"
                    ],
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": [],
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 75.30785179817528,
            "fill_rate": 0.3233748070987654,
            "speedup": 1.6907909277558364
        },
        {
            "annotate": "32",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "transformers_version": "4.4.2",
                    "type_vocab_size": 2,
                    "use_cache": true,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "HasAns_exact": 71.71052631578948,
                    "HasAns_f1": 78.21541157269785,
                    "HasAns_total": 5928,
                    "NoAns_exact": 71.53910849453322,
                    "NoAns_f1": 71.53910849453322,
                    "NoAns_total": 5945,
                    "best_exact": 71.6246946854207,
                    "best_exact_thresh": 0.0,
                    "best_f1": 74.8724804011583,
                    "best_f1_thresh": 0.0,
                    "exact": 71.6246946854207,
                    "f1": 74.8724804011583,
                    "total": 11873
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--1d4e4bc55a9a2bf2/checkpoint-160000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": false,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": false,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "linear_min_parameters": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 5.0,
                    "rewind_model_name_or_path": null
                },
                "speed": {
                    "cuda_eval_elapsed_time": 26.05066195678711,
                    "eval_elapsed_time": 30.27150132902898
                },
                "speedup": 1.6940625797657822,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1449984,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1254912,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2704896,
                            "linear_total": 7077888,
                            "nnz": 2712017,
                            "total": 7087680
                        },
                        "1": {
                            "linear_attention_nnz": 1046528,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1396224,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2442752,
                            "linear_total": 7077888,
                            "nnz": 2449837,
                            "total": 7087488
                        },
                        "10": {
                            "linear_attention_nnz": 722944,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 244224,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 967168,
                            "linear_total": 7077888,
                            "nnz": 972991,
                            "total": 7086912
                        },
                        "11": {
                            "linear_attention_nnz": 476160,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 385536,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 861696,
                            "linear_total": 7077888,
                            "nnz": 867515,
                            "total": 7086912
                        },
                        "2": {
                            "linear_attention_nnz": 1516544,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1474560,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2991104,
                            "linear_total": 7077888,
                            "nnz": 2998432,
                            "total": 7087488
                        },
                        "3": {
                            "linear_attention_nnz": 1539072,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1591296,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3130368,
                            "linear_total": 7077888,
                            "nnz": 3137740,
                            "total": 7087296
                        },
                        "4": {
                            "linear_attention_nnz": 1909760,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1525248,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3435008,
                            "linear_total": 7077888,
                            "nnz": 3442913,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1641472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1411584,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3053056,
                            "linear_total": 7077888,
                            "nnz": 3060503,
                            "total": 7087488
                        },
                        "6": {
                            "linear_attention_nnz": 1680384,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1104384,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2784768,
                            "linear_total": 7077888,
                            "nnz": 2792143,
                            "total": 7087680
                        },
                        "7": {
                            "linear_attention_nnz": 1300480,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 801792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2102272,
                            "linear_total": 7077888,
                            "nnz": 2108938,
                            "total": 7087104
                        },
                        "8": {
                            "linear_attention_nnz": 1438720,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 454656,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1893376,
                            "linear_total": 7077888,
                            "nnz": 1900168,
                            "total": 7087680
                        },
                        "9": {
                            "linear_attention_nnz": 823296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 211968,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1035264,
                            "linear_total": 7077888,
                            "nnz": 1041162,
                            "total": 7086912
                        }
                    },
                    "linear_nnz": 27401728,
                    "linear_sparsity": 67.73787133487654,
                    "linear_total": 84934656,
                    "nnz": 51323081,
                    "pruned_heads": {
                        "0": [
                            4
                        ],
                        "1": [
                            0,
                            5
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            8
                        ],
                        "11": [
                            3,
                            5,
                            7,
                            8,
                            11
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            3
                        ],
                        "7": [
                            3,
                            11,
                            6,
                            7
                        ],
                        "8": [
                            0
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "total": 108887234,
                    "total_sparsity": 52.86584192229551
                },
                "training_args": {
                    "_n_gpu": 1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "length_column_name": "length",
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "mp_parameters": "",
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": [
                        "tensorboard",
                        "wandb"
                    ],
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": [],
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 74.8724804011583,
            "fill_rate": 0.3226212866512346,
            "speedup": 1.6940625797657822
        },
        {
            "annotate": "25",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "transformers_version": "4.4.2",
                    "type_vocab_size": 2,
                    "use_cache": true,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "HasAns_exact": 68.20175438596492,
                    "HasAns_f1": 74.90270003478001,
                    "HasAns_total": 5928,
                    "NoAns_exact": 75.27333894028595,
                    "NoAns_f1": 75.27333894028595,
                    "NoAns_total": 5945,
                    "best_exact": 71.7426092815632,
                    "best_exact_thresh": 0.0,
                    "best_f1": 75.08828483164983,
                    "best_f1_thresh": 0.0,
                    "exact": 71.7426092815632,
                    "f1": 75.08828483164976,
                    "total": 11873
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--333fe38360923d6c/checkpoint-150000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 5,
                    "gelu_patch": false,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": false,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "linear_min_parameters": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10.0,
                    "rewind_model_name_or_path": null
                },
                "speed": {
                    "cuda_eval_elapsed_time": 22.232523056030274,
                    "eval_elapsed_time": 26.513782265014015
                },
                "speedup": 1.9849952022045032,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1149952,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 769536,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1919488,
                            "linear_total": 7077888,
                            "nnz": 1925909,
                            "total": 7087104
                        },
                        "1": {
                            "linear_attention_nnz": 760832,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 973824,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1734656,
                            "linear_total": 7077888,
                            "nnz": 1740794,
                            "total": 7086528
                        },
                        "10": {
                            "linear_attention_nnz": 590848,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 175104,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 765952,
                            "linear_total": 7077888,
                            "nnz": 771538,
                            "total": 7086720
                        },
                        "11": {
                            "linear_attention_nnz": 390144,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 264192,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 654336,
                            "linear_total": 7077888,
                            "nnz": 659916,
                            "total": 7086720
                        },
                        "2": {
                            "linear_attention_nnz": 1390592,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1055232,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2445824,
                            "linear_total": 7077888,
                            "nnz": 2452719,
                            "total": 7087296
                        },
                        "3": {
                            "linear_attention_nnz": 1472512,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1092096,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2564608,
                            "linear_total": 7077888,
                            "nnz": 2571623,
                            "total": 7087296
                        },
                        "4": {
                            "linear_attention_nnz": 1591296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1058304,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2649600,
                            "linear_total": 7077888,
                            "nnz": 2656881,
                            "total": 7087680
                        },
                        "5": {
                            "linear_attention_nnz": 1536000,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1004544,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2540544,
                            "linear_total": 7077888,
                            "nnz": 2547726,
                            "total": 7087488
                        },
                        "6": {
                            "linear_attention_nnz": 1400832,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 768000,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2168832,
                            "linear_total": 7077888,
                            "nnz": 2175764,
                            "total": 7087488
                        },
                        "7": {
                            "linear_attention_nnz": 1145856,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 577536,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1723392,
                            "linear_total": 7077888,
                            "nnz": 1729848,
                            "total": 7087104
                        },
                        "8": {
                            "linear_attention_nnz": 1049600,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 331776,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1381376,
                            "linear_total": 7077888,
                            "nnz": 1387608,
                            "total": 7087104
                        },
                        "9": {
                            "linear_attention_nnz": 712704,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 138240,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 850944,
                            "linear_total": 7077888,
                            "nnz": 856730,
                            "total": 7086912
                        }
                    },
                    "linear_nnz": 21399552,
                    "linear_sparsity": 74.8046875,
                    "linear_total": 84934656,
                    "nnz": 45315778,
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            5,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            3,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            2
                        ],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            2,
                            3
                        ],
                        "7": [
                            3,
                            11,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "total": 108884162,
                    "total_sparsity": 58.38166252315007
                },
                "training_args": {
                    "_n_gpu": 1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "length_column_name": "length",
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "mp_parameters": "",
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": [
                        "tensorboard",
                        "wandb"
                    ],
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": [],
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 75.08828483164976,
            "fill_rate": 0.251953125,
            "speedup": 1.9849952022045032
        },
        {
            "annotate": "24",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "transformers_version": "4.4.2",
                    "type_vocab_size": 2,
                    "use_cache": true,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "HasAns_exact": 70.4284750337382,
                    "HasAns_f1": 76.73198962499656,
                    "HasAns_total": 5928,
                    "NoAns_exact": 72.56518082422204,
                    "NoAns_f1": 72.56518082422204,
                    "NoAns_total": 5945,
                    "best_exact": 71.49835761812515,
                    "best_exact_thresh": 0.0,
                    "best_f1": 74.64560216432078,
                    "best_f1_thresh": 0.0,
                    "exact": 71.49835761812515,
                    "f1": 74.64560216432078,
                    "total": 11873
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--333fe38360923d6c/checkpoint-160000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 5,
                    "gelu_patch": false,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": false,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "linear_min_parameters": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10.0,
                    "rewind_model_name_or_path": null
                },
                "speed": {
                    "cuda_eval_elapsed_time": 22.191959045410158,
                    "eval_elapsed_time": 26.45285324798897
                },
                "speedup": 1.988623514887414,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1136640,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 764928,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1901568,
                            "linear_total": 7077888,
                            "nnz": 1907986,
                            "total": 7087104
                        },
                        "1": {
                            "linear_attention_nnz": 766976,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 970752,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1737728,
                            "linear_total": 7077888,
                            "nnz": 1743864,
                            "total": 7086528
                        },
                        "10": {
                            "linear_attention_nnz": 575488,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 175104,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 750592,
                            "linear_total": 7077888,
                            "nnz": 756178,
                            "total": 7086720
                        },
                        "11": {
                            "linear_attention_nnz": 363520,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 264192,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 627712,
                            "linear_total": 7077888,
                            "nnz": 633260,
                            "total": 7086720
                        },
                        "2": {
                            "linear_attention_nnz": 1369088,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1049088,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2418176,
                            "linear_total": 7077888,
                            "nnz": 2425067,
                            "total": 7087296
                        },
                        "3": {
                            "linear_attention_nnz": 1461248,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1089024,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2550272,
                            "linear_total": 7077888,
                            "nnz": 2557285,
                            "total": 7087296
                        },
                        "4": {
                            "linear_attention_nnz": 1597440,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1056768,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2654208,
                            "linear_total": 7077888,
                            "nnz": 2661488,
                            "total": 7087680
                        },
                        "5": {
                            "linear_attention_nnz": 1525760,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 993792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2519552,
                            "linear_total": 7077888,
                            "nnz": 2526727,
                            "total": 7087488
                        },
                        "6": {
                            "linear_attention_nnz": 1401856,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 768000,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2169856,
                            "linear_total": 7077888,
                            "nnz": 2176788,
                            "total": 7087488
                        },
                        "7": {
                            "linear_attention_nnz": 1100800,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 576000,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1676800,
                            "linear_total": 7077888,
                            "nnz": 1683223,
                            "total": 7087104
                        },
                        "8": {
                            "linear_attention_nnz": 1041408,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 330240,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1371648,
                            "linear_total": 7077888,
                            "nnz": 1377879,
                            "total": 7087104
                        },
                        "9": {
                            "linear_attention_nnz": 704512,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 138240,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 842752,
                            "linear_total": 7077888,
                            "nnz": 848538,
                            "total": 7086912
                        }
                    },
                    "linear_nnz": 21220864,
                    "linear_sparsity": 75.0150704089506,
                    "linear_total": 84934656,
                    "nnz": 45137005,
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            5,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            3,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            2
                        ],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            2,
                            3
                        ],
                        "7": [
                            3,
                            11,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "total": 108884162,
                    "total_sparsity": 58.54584893623005
                },
                "training_args": {
                    "_n_gpu": 1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "length_column_name": "length",
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "mp_parameters": "",
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": [
                        "tensorboard",
                        "wandb"
                    ],
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": [],
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 74.64560216432078,
            "fill_rate": 0.24984929591049398,
            "speedup": 1.988623514887414
        },
        {
            "annotate": "34",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "transformers_version": "4.4.2",
                    "type_vocab_size": 2,
                    "use_cache": true,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "HasAns_exact": 71.2719298245614,
                    "HasAns_f1": 78.12614467791248,
                    "HasAns_total": 5928,
                    "NoAns_exact": 74.61732548359966,
                    "NoAns_f1": 74.61732548359966,
                    "NoAns_total": 5945,
                    "best_exact": 72.9470226564474,
                    "best_exact_thresh": 0.0,
                    "best_f1": 76.36922308183821,
                    "best_f1_thresh": 0.0,
                    "exact": 72.9470226564474,
                    "f1": 76.36922308183819,
                    "total": 11873
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de/checkpoint-135000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 5,
                    "gelu_patch": false,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": false,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "linear_min_parameters": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 5.0,
                    "rewind_model_name_or_path": null
                },
                "speed": {
                    "cuda_eval_elapsed_time": 26.076661651611328,
                    "eval_elapsed_time": 30.333236434031278
                },
                "speedup": 1.6923735173130998,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1624064,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1434624,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3058688,
                            "linear_total": 7077888,
                            "nnz": 3066150,
                            "total": 7087680
                        },
                        "1": {
                            "linear_attention_nnz": 1034240,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1509888,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2544128,
                            "linear_total": 7077888,
                            "nnz": 2551127,
                            "total": 7087104
                        },
                        "10": {
                            "linear_attention_nnz": 760832,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 251904,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1012736,
                            "linear_total": 7077888,
                            "nnz": 1018692,
                            "total": 7087104
                        },
                        "11": {
                            "linear_attention_nnz": 533504,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 417792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 951296,
                            "linear_total": 7077888,
                            "nnz": 957200,
                            "total": 7087104
                        },
                        "2": {
                            "linear_attention_nnz": 1571840,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1566720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3138560,
                            "linear_total": 7077888,
                            "nnz": 3145948,
                            "total": 7087488
                        },
                        "3": {
                            "linear_attention_nnz": 1561600,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1681920,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3243520,
                            "linear_total": 7077888,
                            "nnz": 3250951,
                            "total": 7087296
                        },
                        "4": {
                            "linear_attention_nnz": 1954816,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1637376,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3592192,
                            "linear_total": 7077888,
                            "nnz": 3600170,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1661952,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1532928,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3194880,
                            "linear_total": 7077888,
                            "nnz": 3202406,
                            "total": 7087488
                        },
                        "6": {
                            "linear_attention_nnz": 1730560,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1167360,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2897920,
                            "linear_total": 7077888,
                            "nnz": 2905400,
                            "total": 7087680
                        },
                        "7": {
                            "linear_attention_nnz": 1313792,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 892416,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2206208,
                            "linear_total": 7077888,
                            "nnz": 2212933,
                            "total": 7087104
                        },
                        "8": {
                            "linear_attention_nnz": 1441792,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 473088,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1914880,
                            "linear_total": 7077888,
                            "nnz": 1921620,
                            "total": 7087488
                        },
                        "9": {
                            "linear_attention_nnz": 906240,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 222720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1128960,
                            "linear_total": 7077888,
                            "nnz": 1134961,
                            "total": 7086912
                        }
                    },
                    "linear_nnz": 28883968,
                    "linear_sparsity": 65.99271797839505,
                    "linear_total": 84934656,
                    "nnz": 52806280,
                    "pruned_heads": {
                        "0": [
                            4
                        ],
                        "1": [
                            0,
                            8,
                            2,
                            5
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6
                        ],
                        "11": [
                            3,
                            11,
                            5,
                            7
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            3
                        ],
                        "7": [
                            3,
                            11,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            8
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "total": 108887042,
                    "total_sparsity": 51.50361417660698
                },
                "training_args": {
                    "_n_gpu": 1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "length_column_name": "length",
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "mp_parameters": "",
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": [
                        "tensorboard",
                        "wandb"
                    ],
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": [],
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 76.36922308183819,
            "fill_rate": 0.34007282021604945,
            "speedup": 1.6923735173130998
        },
        {
            "annotate": "32",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "transformers_version": "4.4.2",
                    "type_vocab_size": 2,
                    "use_cache": true,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "HasAns_exact": 70.39473684210526,
                    "HasAns_f1": 77.14341107371645,
                    "HasAns_total": 5928,
                    "NoAns_exact": 74.02859545836837,
                    "NoAns_f1": 74.02859545836837,
                    "NoAns_total": 5945,
                    "best_exact": 72.21426766613324,
                    "best_exact_thresh": 0.0,
                    "best_f1": 75.58377333824572,
                    "best_f1_thresh": 0.0,
                    "exact": 72.21426766613324,
                    "f1": 75.58377333824575,
                    "total": 11873
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de/checkpoint-160000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 5,
                    "gelu_patch": false,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": false,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "linear_min_parameters": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 5.0,
                    "rewind_model_name_or_path": null
                },
                "speed": {
                    "cuda_eval_elapsed_time": 25.73425926208496,
                    "eval_elapsed_time": 30.011940322001465
                },
                "speedup": 1.7148910776748587,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1526784,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1353216,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2880000,
                            "linear_total": 7077888,
                            "nnz": 2887377,
                            "total": 7087680
                        },
                        "1": {
                            "linear_attention_nnz": 974848,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1456128,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2430976,
                            "linear_total": 7077888,
                            "nnz": 2437844,
                            "total": 7086912
                        },
                        "10": {
                            "linear_attention_nnz": 769024,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 245760,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1014784,
                            "linear_total": 7077888,
                            "nnz": 1020736,
                            "total": 7087104
                        },
                        "11": {
                            "linear_attention_nnz": 494592,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 403968,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 898560,
                            "linear_total": 7077888,
                            "nnz": 904423,
                            "total": 7087104
                        },
                        "2": {
                            "linear_attention_nnz": 1556480,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1522176,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3078656,
                            "linear_total": 7077888,
                            "nnz": 3086015,
                            "total": 7087488
                        },
                        "3": {
                            "linear_attention_nnz": 1564672,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1635840,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3200512,
                            "linear_total": 7077888,
                            "nnz": 3207913,
                            "total": 7087296
                        },
                        "4": {
                            "linear_attention_nnz": 1926144,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1585152,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3511296,
                            "linear_total": 7077888,
                            "nnz": 3519240,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1669120,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1482240,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3151360,
                            "linear_total": 7077888,
                            "nnz": 3158853,
                            "total": 7087488
                        },
                        "6": {
                            "linear_attention_nnz": 1571840,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1128960,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2700800,
                            "linear_total": 7077888,
                            "nnz": 2708063,
                            "total": 7087488
                        },
                        "7": {
                            "linear_attention_nnz": 1311744,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 870912,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2182656,
                            "linear_total": 7077888,
                            "nnz": 2189367,
                            "total": 7087104
                        },
                        "8": {
                            "linear_attention_nnz": 1420288,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 460800,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1881088,
                            "linear_total": 7077888,
                            "nnz": 1887788,
                            "total": 7087488
                        },
                        "9": {
                            "linear_attention_nnz": 858112,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 216576,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1074688,
                            "linear_total": 7077888,
                            "nnz": 1080621,
                            "total": 7086912
                        }
                    },
                    "linear_nnz": 28005376,
                    "linear_sparsity": 67.02715084876543,
                    "linear_total": 84934656,
                    "nnz": 51926962,
                    "pruned_heads": {
                        "0": [
                            4
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6
                        ],
                        "11": [
                            3,
                            11,
                            5,
                            7
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            2,
                            3
                        ],
                        "7": [
                            3,
                            11,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            8
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "total": 108886658,
                    "total_sparsity": 52.31099663284734
                },
                "training_args": {
                    "_n_gpu": 1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "length_column_name": "length",
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "mp_parameters": "",
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": [
                        "tensorboard",
                        "wandb"
                    ],
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": [],
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 75.58377333824575,
            "fill_rate": 0.32972849151234573,
            "speedup": 1.7148910776748587
        },
        {
            "annotate": "19",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "transformers_version": "4.4.2",
                    "type_vocab_size": 2,
                    "use_cache": true,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "HasAns_exact": 69.26450742240216,
                    "HasAns_f1": 76.47931310832281,
                    "HasAns_total": 5928,
                    "NoAns_exact": 71.15222876366694,
                    "NoAns_f1": 71.15222876366694,
                    "NoAns_total": 5945,
                    "best_exact": 70.2097195317106,
                    "best_exact_thresh": 0.0,
                    "best_f1": 73.81195722278605,
                    "best_f1_thresh": 0.0,
                    "exact": 70.2097195317106,
                    "f1": 73.81195722278603,
                    "total": 11873
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--489923cf6100d0c5/checkpoint-140000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 5,
                    "gelu_patch": false,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": false,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "linear_min_parameters": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20.0,
                    "rewind_model_name_or_path": null
                },
                "speed": {
                    "cuda_eval_elapsed_time": 20.692826644897462,
                    "eval_elapsed_time": 24.999257856048644
                },
                "speedup": 2.13269324469034,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 957440,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 419328,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1376768,
                            "linear_total": 7077888,
                            "nnz": 1382833,
                            "total": 7086912
                        },
                        "1": {
                            "linear_attention_nnz": 748544,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 588288,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1336832,
                            "linear_total": 7077888,
                            "nnz": 1342719,
                            "total": 7086528
                        },
                        "10": {
                            "linear_attention_nnz": 463872,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 127488,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 591360,
                            "linear_total": 7077888,
                            "nnz": 596851,
                            "total": 7086720
                        },
                        "11": {
                            "linear_attention_nnz": 324608,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 162816,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 487424,
                            "linear_total": 7077888,
                            "nnz": 492842,
                            "total": 7086720
                        },
                        "2": {
                            "linear_attention_nnz": 1277952,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 645120,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1923072,
                            "linear_total": 7077888,
                            "nnz": 1929668,
                            "total": 7087296
                        },
                        "3": {
                            "linear_attention_nnz": 1306624,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 688128,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1994752,
                            "linear_total": 7077888,
                            "nnz": 2001408,
                            "total": 7087296
                        },
                        "4": {
                            "linear_attention_nnz": 1461248,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 746496,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2207744,
                            "linear_total": 7077888,
                            "nnz": 2214694,
                            "total": 7087680
                        },
                        "5": {
                            "linear_attention_nnz": 1142784,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 660480,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1803264,
                            "linear_total": 7077888,
                            "nnz": 1809806,
                            "total": 7087104
                        },
                        "6": {
                            "linear_attention_nnz": 1047552,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 486912,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1534464,
                            "linear_total": 7077888,
                            "nnz": 1540861,
                            "total": 7087296
                        },
                        "7": {
                            "linear_attention_nnz": 941056,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 391680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1332736,
                            "linear_total": 7077888,
                            "nnz": 1338911,
                            "total": 7087104
                        },
                        "8": {
                            "linear_attention_nnz": 935936,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 227328,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1163264,
                            "linear_total": 7077888,
                            "nnz": 1169428,
                            "total": 7087296
                        },
                        "9": {
                            "linear_attention_nnz": 556032,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 105984,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 662016,
                            "linear_total": 7077888,
                            "nnz": 667621,
                            "total": 7086720
                        }
                    },
                    "linear_nnz": 16413696,
                    "linear_sparsity": 80.67491319444444,
                    "linear_total": 84934656,
                    "nnz": 40326364,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            3,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            2
                        ],
                        "5": [
                            1,
                            2,
                            11,
                            6
                        ],
                        "6": [
                            3,
                            10,
                            2
                        ],
                        "7": [
                            3,
                            11,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            8,
                            4
                        ],
                        "9": [
                            1,
                            2,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "total": 108883394,
                    "total_sparsity": 62.96371510976228
                },
                "training_args": {
                    "_n_gpu": 1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "length_column_name": "length",
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "mp_parameters": "",
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": [
                        "tensorboard",
                        "wandb"
                    ],
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": [],
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 73.81195722278603,
            "fill_rate": 0.19325086805555558,
            "speedup": 2.13269324469034
        },
        {
            "annotate": "18",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "transformers_version": "4.4.2",
                    "type_vocab_size": 2,
                    "use_cache": true,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "HasAns_exact": 68.3029689608637,
                    "HasAns_f1": 74.93622452026204,
                    "HasAns_total": 5928,
                    "NoAns_exact": 72.63246425567704,
                    "NoAns_f1": 72.63246425567704,
                    "NoAns_total": 5945,
                    "best_exact": 70.47081613745473,
                    "best_exact_thresh": 0.0,
                    "best_f1": 73.78269510284802,
                    "best_f1_thresh": 0.0,
                    "exact": 70.47081613745473,
                    "f1": 73.78269510284804,
                    "total": 11873
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--489923cf6100d0c5/checkpoint-155000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 5,
                    "gelu_patch": false,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": false,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "linear_min_parameters": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20.0,
                    "rewind_model_name_or_path": null
                },
                "speed": {
                    "cuda_eval_elapsed_time": 20.5203953704834,
                    "eval_elapsed_time": 24.813339828979224
                },
                "speedup": 2.150614098917407,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 935936,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 402432,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1338368,
                            "linear_total": 7077888,
                            "nnz": 1344326,
                            "total": 7086720
                        },
                        "1": {
                            "linear_attention_nnz": 718848,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 569856,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1288704,
                            "linear_total": 7077888,
                            "nnz": 1294579,
                            "total": 7086528
                        },
                        "10": {
                            "linear_attention_nnz": 464896,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 122880,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 587776,
                            "linear_total": 7077888,
                            "nnz": 593296,
                            "total": 7086720
                        },
                        "11": {
                            "linear_attention_nnz": 316416,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 156672,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 473088,
                            "linear_total": 7077888,
                            "nnz": 478502,
                            "total": 7086720
                        },
                        "2": {
                            "linear_attention_nnz": 1244160,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 631296,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1875456,
                            "linear_total": 7077888,
                            "nnz": 1882043,
                            "total": 7087296
                        },
                        "3": {
                            "linear_attention_nnz": 1270784,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 680448,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1951232,
                            "linear_total": 7077888,
                            "nnz": 1957851,
                            "total": 7087296
                        },
                        "4": {
                            "linear_attention_nnz": 1404928,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 734208,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2139136,
                            "linear_total": 7077888,
                            "nnz": 2146078,
                            "total": 7087680
                        },
                        "5": {
                            "linear_attention_nnz": 1121280,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 652800,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1774080,
                            "linear_total": 7077888,
                            "nnz": 1780617,
                            "total": 7087104
                        },
                        "6": {
                            "linear_attention_nnz": 1036288,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 473088,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1509376,
                            "linear_total": 7077888,
                            "nnz": 1515732,
                            "total": 7087296
                        },
                        "7": {
                            "linear_attention_nnz": 925696,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 388608,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1314304,
                            "linear_total": 7077888,
                            "nnz": 1320413,
                            "total": 7087104
                        },
                        "8": {
                            "linear_attention_nnz": 918528,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 227328,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1145856,
                            "linear_total": 7077888,
                            "nnz": 1151988,
                            "total": 7087296
                        },
                        "9": {
                            "linear_attention_nnz": 528384,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 105984,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 634368,
                            "linear_total": 7077888,
                            "nnz": 639941,
                            "total": 7086720
                        }
                    },
                    "linear_nnz": 16031744,
                    "linear_sparsity": 81.12461419753086,
                    "linear_total": 84934656,
                    "nnz": 39944088,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            3,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            2
                        ],
                        "5": [
                            1,
                            2,
                            11,
                            6
                        ],
                        "6": [
                            3,
                            10,
                            2
                        ],
                        "7": [
                            3,
                            11,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            8,
                            4
                        ],
                        "9": [
                            1,
                            2,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "total": 108883202,
                    "total_sparsity": 63.31473793358868
                },
                "training_args": {
                    "_n_gpu": 1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "length_column_name": "length",
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "mp_parameters": "",
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": [
                        "tensorboard",
                        "wandb"
                    ],
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": [],
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 73.78269510284804,
            "fill_rate": 0.18875385802469136,
            "speedup": 2.150614098917407
        },
        {
            "annotate": "18",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "transformers_version": "4.4.2",
                    "type_vocab_size": 2,
                    "use_cache": true,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "HasAns_exact": 69.73684210526316,
                    "HasAns_f1": 77.18943506323211,
                    "HasAns_total": 5928,
                    "NoAns_exact": 68.19175777964676,
                    "NoAns_f1": 68.19175777964676,
                    "NoAns_total": 5945,
                    "best_exact": 68.96319380106124,
                    "best_exact_thresh": 0.0,
                    "best_f1": 72.68415489386358,
                    "best_f1_thresh": 0.0,
                    "exact": 68.96319380106124,
                    "f1": 72.68415489386355,
                    "total": 11873
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--489923cf6100d0c5/checkpoint-160000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 5,
                    "gelu_patch": false,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": false,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "linear_min_parameters": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20.0,
                    "rewind_model_name_or_path": null
                },
                "speed": {
                    "cuda_eval_elapsed_time": 20.526136947631837,
                    "eval_elapsed_time": 24.803041459061205
                },
                "speedup": 2.1500125285002873,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 912384,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 400896,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1313280,
                            "linear_total": 7077888,
                            "nnz": 1319237,
                            "total": 7086720
                        },
                        "1": {
                            "linear_attention_nnz": 740352,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 568320,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1308672,
                            "linear_total": 7077888,
                            "nnz": 1314546,
                            "total": 7086528
                        },
                        "10": {
                            "linear_attention_nnz": 468992,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 122880,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 591872,
                            "linear_total": 7077888,
                            "nnz": 597296,
                            "total": 7086720
                        },
                        "11": {
                            "linear_attention_nnz": 297984,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 155136,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 453120,
                            "linear_total": 7077888,
                            "nnz": 458533,
                            "total": 7086720
                        },
                        "2": {
                            "linear_attention_nnz": 1220608,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 629760,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1850368,
                            "linear_total": 7077888,
                            "nnz": 1856890,
                            "total": 7087296
                        },
                        "3": {
                            "linear_attention_nnz": 1288192,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 680448,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1968640,
                            "linear_total": 7077888,
                            "nnz": 1975259,
                            "total": 7087296
                        },
                        "4": {
                            "linear_attention_nnz": 1471488,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 734208,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2205696,
                            "linear_total": 7077888,
                            "nnz": 2212638,
                            "total": 7087680
                        },
                        "5": {
                            "linear_attention_nnz": 1118208,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 652800,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1771008,
                            "linear_total": 7077888,
                            "nnz": 1777545,
                            "total": 7087104
                        },
                        "6": {
                            "linear_attention_nnz": 998400,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 473088,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1471488,
                            "linear_total": 7077888,
                            "nnz": 1477844,
                            "total": 7087296
                        },
                        "7": {
                            "linear_attention_nnz": 930816,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 388608,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1319424,
                            "linear_total": 7077888,
                            "nnz": 1325533,
                            "total": 7087104
                        },
                        "8": {
                            "linear_attention_nnz": 894976,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 225792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1120768,
                            "linear_total": 7077888,
                            "nnz": 1126899,
                            "total": 7087296
                        },
                        "9": {
                            "linear_attention_nnz": 527360,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 105984,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 633344,
                            "linear_total": 7077888,
                            "nnz": 638949,
                            "total": 7086720
                        }
                    },
                    "linear_nnz": 16007680,
                    "linear_sparsity": 81.15294656635803,
                    "linear_total": 84934656,
                    "nnz": 39919891,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            3,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            2
                        ],
                        "5": [
                            1,
                            2,
                            11,
                            6
                        ],
                        "6": [
                            3,
                            10,
                            2
                        ],
                        "7": [
                            3,
                            11,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            8,
                            4
                        ],
                        "9": [
                            1,
                            2,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "total": 108883202,
                    "total_sparsity": 63.336960828907294
                },
                "training_args": {
                    "_n_gpu": 1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "length_column_name": "length",
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "mp_parameters": "",
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": [
                        "tensorboard",
                        "wandb"
                    ],
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": [],
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 72.68415489386355,
            "fill_rate": 0.1884705343364197,
            "speedup": 2.1500125285002873
        },
        {
            "annotate": "18",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "transformers_version": "4.4.2",
                    "type_vocab_size": 2,
                    "use_cache": true,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "HasAns_exact": 66.95344129554655,
                    "HasAns_f1": 73.95621625172882,
                    "HasAns_total": 5928,
                    "NoAns_exact": 73.86038687973087,
                    "NoAns_f1": 73.86038687973087,
                    "NoAns_total": 5945,
                    "best_exact": 70.41185883938347,
                    "best_exact_thresh": 0.0,
                    "best_f1": 73.90823296051964,
                    "best_f1_thresh": 0.0,
                    "exact": 70.41185883938347,
                    "f1": 73.90823296051956,
                    "total": 11873
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682/checkpoint-155000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": false,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": false,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "linear_min_parameters": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20.0,
                    "rewind_model_name_or_path": null
                },
                "speed": {
                    "cuda_eval_elapsed_time": 20.151872497558593,
                    "eval_elapsed_time": 24.383727533044294
                },
                "speedup": 2.1899429745036167,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 924672,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 396288,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1320960,
                            "linear_total": 7077888,
                            "nnz": 1326914,
                            "total": 7086720
                        },
                        "1": {
                            "linear_attention_nnz": 711680,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 560640,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1272320,
                            "linear_total": 7077888,
                            "nnz": 1278125,
                            "total": 7086528
                        },
                        "10": {
                            "linear_attention_nnz": 423936,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 118272,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 542208,
                            "linear_total": 7077888,
                            "nnz": 547629,
                            "total": 7086528
                        },
                        "11": {
                            "linear_attention_nnz": 314368,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 152064,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 466432,
                            "linear_total": 7077888,
                            "nnz": 471875,
                            "total": 7086720
                        },
                        "2": {
                            "linear_attention_nnz": 1217536,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 599040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1816576,
                            "linear_total": 7077888,
                            "nnz": 1823078,
                            "total": 7087296
                        },
                        "3": {
                            "linear_attention_nnz": 1281024,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 646656,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1927680,
                            "linear_total": 7077888,
                            "nnz": 1934245,
                            "total": 7087296
                        },
                        "4": {
                            "linear_attention_nnz": 1203200,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 672768,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1875968,
                            "linear_total": 7077888,
                            "nnz": 1882582,
                            "total": 7087296
                        },
                        "5": {
                            "linear_attention_nnz": 979968,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 609792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1589760,
                            "linear_total": 7077888,
                            "nnz": 1596077,
                            "total": 7086912
                        },
                        "6": {
                            "linear_attention_nnz": 1017856,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 486912,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1504768,
                            "linear_total": 7077888,
                            "nnz": 1511069,
                            "total": 7087296
                        },
                        "7": {
                            "linear_attention_nnz": 886784,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 374784,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1261568,
                            "linear_total": 7077888,
                            "nnz": 1267636,
                            "total": 7086912
                        },
                        "8": {
                            "linear_attention_nnz": 891904,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 224256,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1116160,
                            "linear_total": 7077888,
                            "nnz": 1122322,
                            "total": 7087296
                        },
                        "9": {
                            "linear_attention_nnz": 510976,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 101376,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 612352,
                            "linear_total": 7077888,
                            "nnz": 617890,
                            "total": 7086720
                        }
                    },
                    "linear_nnz": 15306752,
                    "linear_sparsity": 81.97820216049382,
                    "linear_total": 84934656,
                    "nnz": 39218164,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            3,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            8,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            3,
                            10,
                            2
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            4
                        ],
                        "9": [
                            1,
                            2,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "total": 108882242,
                    "total_sparsity": 63.98112008016881
                },
                "training_args": {
                    "_n_gpu": 1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "length_column_name": "length",
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "mp_parameters": "",
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": [
                        "tensorboard",
                        "wandb"
                    ],
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": [],
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 73.90823296051956,
            "fill_rate": 0.18021797839506182,
            "speedup": 2.1899429745036167
        },
        {
            "annotate": "17",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "transformers_version": "4.4.2",
                    "type_vocab_size": 2,
                    "use_cache": true,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "HasAns_exact": 68.31983805668017,
                    "HasAns_f1": 75.3064164201114,
                    "HasAns_total": 5928,
                    "NoAns_exact": 70.58031959629942,
                    "NoAns_f1": 70.58031959629942,
                    "NoAns_total": 5945,
                    "best_exact": 69.45169712793734,
                    "best_exact_thresh": 0.0,
                    "best_f1": 72.93998454800146,
                    "best_f1_thresh": 0.0,
                    "exact": 69.45169712793734,
                    "f1": 72.93998454800146,
                    "total": 11873
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682/checkpoint-160000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 10,
                    "gelu_patch": false,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": false,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "linear_min_parameters": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20.0,
                    "rewind_model_name_or_path": null
                },
                "speed": {
                    "cuda_eval_elapsed_time": 19.807459594726563,
                    "eval_elapsed_time": 24.067060329020023
                },
                "speedup": 2.2280217908848052,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 920576,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 396288,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1316864,
                            "linear_total": 7077888,
                            "nnz": 1322818,
                            "total": 7086720
                        },
                        "1": {
                            "linear_attention_nnz": 698368,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 559104,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1257472,
                            "linear_total": 7077888,
                            "nnz": 1263276,
                            "total": 7086528
                        },
                        "10": {
                            "linear_attention_nnz": 433152,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 118272,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 551424,
                            "linear_total": 7077888,
                            "nnz": 556877,
                            "total": 7086528
                        },
                        "11": {
                            "linear_attention_nnz": 291840,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 152064,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 443904,
                            "linear_total": 7077888,
                            "nnz": 449347,
                            "total": 7086720
                        },
                        "2": {
                            "linear_attention_nnz": 1227776,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 599040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1826816,
                            "linear_total": 7077888,
                            "nnz": 1833318,
                            "total": 7087296
                        },
                        "3": {
                            "linear_attention_nnz": 1266688,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 643584,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1910272,
                            "linear_total": 7077888,
                            "nnz": 1916835,
                            "total": 7087296
                        },
                        "4": {
                            "linear_attention_nnz": 1227776,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 672768,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1900544,
                            "linear_total": 7077888,
                            "nnz": 1907158,
                            "total": 7087296
                        },
                        "5": {
                            "linear_attention_nnz": 965632,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 609792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1575424,
                            "linear_total": 7077888,
                            "nnz": 1581741,
                            "total": 7086912
                        },
                        "6": {
                            "linear_attention_nnz": 967680,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 486912,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1454592,
                            "linear_total": 7077888,
                            "nnz": 1460893,
                            "total": 7087296
                        },
                        "7": {
                            "linear_attention_nnz": 862208,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 374784,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1236992,
                            "linear_total": 7077888,
                            "nnz": 1243060,
                            "total": 7086912
                        },
                        "8": {
                            "linear_attention_nnz": 893952,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 224256,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1118208,
                            "linear_total": 7077888,
                            "nnz": 1124370,
                            "total": 7087296
                        },
                        "9": {
                            "linear_attention_nnz": 500736,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 101376,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 602112,
                            "linear_total": 7077888,
                            "nnz": 607650,
                            "total": 7086720
                        }
                    },
                    "linear_nnz": 15194624,
                    "linear_sparsity": 82.11021894290124,
                    "linear_total": 84934656,
                    "nnz": 39106065,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            3,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            8,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            3,
                            10,
                            2
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            4
                        ],
                        "9": [
                            1,
                            2,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "total": 108882242,
                    "total_sparsity": 64.08407442602073
                },
                "training_args": {
                    "_n_gpu": 1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "length_column_name": "length",
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "mp_parameters": "",
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": [
                        "tensorboard",
                        "wandb"
                    ],
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": [],
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 72.93998454800146,
            "fill_rate": 0.17889781057098764,
            "speedup": 2.2280217908848052
        }
    ],
    "Block/struct method, final fine tuned, s=b": [
        {
            "annotate": "32",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            5,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            3,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            2
                        ],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            2,
                            3
                        ],
                        "7": [
                            3,
                            11,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "transformers_version": "4.5.1",
                    "type_vocab_size": 2,
                    "use_cache": true,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "HasAns_exact": 71.84547908232119,
                    "HasAns_f1": 78.27031796244835,
                    "HasAns_total": 5928,
                    "NoAns_exact": 73.00252312867956,
                    "NoAns_f1": 73.00252312867956,
                    "NoAns_total": 5945,
                    "best_exact": 72.42482944495914,
                    "best_exact_thresh": 0.0,
                    "best_f1": 75.63264927831177,
                    "best_f1_thresh": 0.0,
                    "exact": 72.42482944495914,
                    "f1": 75.63264927831169,
                    "total": 11873
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--333fe38360923d6c/checkpoint-60000",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--333fe38360923d6c/checkpoint-150000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                    "distil_temperature": 2.0,
                    "eval_with_current_patch_params": false,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "gelu_patch": false,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "layer_norm_patch": false,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "linear_min_parameters": 0.005,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "qat": false,
                    "qconfig": "default",
                    "regularization": "",
                    "regularization_final_lambda": 0,
                    "rewind_model_name_or_path": null
                },
                "speed": {
                    "cuda_eval_elapsed_time": 22.437649841308595,
                    "eval_elapsed_time": 26.890222856076434
                },
                "speedup": 1.9668482176717705,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 769536,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2342400,
                            "linear_total": 7077888,
                            "nnz": 2349045,
                            "total": 7087104
                        },
                        "1": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 973824,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1956864,
                            "linear_total": 7077888,
                            "nnz": 1963066,
                            "total": 7086528
                        },
                        "10": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 175104,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1354752,
                            "linear_total": 7077888,
                            "nnz": 1360626,
                            "total": 7086720
                        },
                        "11": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 264192,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1443840,
                            "linear_total": 7077888,
                            "nnz": 1449772,
                            "total": 7086720
                        },
                        "2": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1055232,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2824704,
                            "linear_total": 7077888,
                            "nnz": 2831727,
                            "total": 7087296
                        },
                        "3": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1092096,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2861568,
                            "linear_total": 7077888,
                            "nnz": 2868615,
                            "total": 7087296
                        },
                        "4": {
                            "linear_attention_nnz": 2162688,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1058304,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3220992,
                            "linear_total": 7077888,
                            "nnz": 3228401,
                            "total": 7087680
                        },
                        "5": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1004544,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2970624,
                            "linear_total": 7077888,
                            "nnz": 2977806,
                            "total": 7087488
                        },
                        "6": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 768000,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2734080,
                            "linear_total": 7077888,
                            "nnz": 2741108,
                            "total": 7087488
                        },
                        "7": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 577536,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2150400,
                            "linear_total": 7077888,
                            "nnz": 2156920,
                            "total": 7087104
                        },
                        "8": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 331776,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1904640,
                            "linear_total": 7077888,
                            "nnz": 1911000,
                            "total": 7087104
                        },
                        "9": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 138240,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1514496,
                            "linear_total": 7077888,
                            "nnz": 1520538,
                            "total": 7086912
                        }
                    },
                    "linear_nnz": 27279360,
                    "linear_sparsity": 67.88194444444444,
                    "linear_total": 84934656,
                    "nnz": 51197346,
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            5,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            3,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            2
                        ],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            2,
                            3
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "total": 108884162,
                    "total_sparsity": 52.97998803535816
                },
                "training_args": {
                    "_n_gpu": 1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "length_column_name": "length",
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--333fe38360923d6c",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "mp_parameters": "",
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--333fe38360923d6c",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": [
                        "tensorboard",
                        "wandb"
                    ],
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--333fe38360923d6c",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": [],
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 75.63264927831169,
            "fill_rate": 0.3211805555555556,
            "speedup": 1.9668482176717705
        },
        {
            "annotate": "32",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            5,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            3,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            2
                        ],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            2,
                            3
                        ],
                        "7": [
                            3,
                            11,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "transformers_version": "4.5.1",
                    "type_vocab_size": 2,
                    "use_cache": true,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "HasAns_exact": 71.94669365721997,
                    "HasAns_f1": 78.3671658213592,
                    "HasAns_total": 5928,
                    "NoAns_exact": 72.54835996635829,
                    "NoAns_f1": 72.54835996635829,
                    "NoAns_total": 5945,
                    "best_exact": 72.24795755074538,
                    "best_exact_thresh": 0.0,
                    "best_f1": 75.4535971522799,
                    "best_f1_thresh": 0.0,
                    "exact": 72.24795755074538,
                    "f1": 75.45359715227984,
                    "total": 11873
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--333fe38360923d6c/checkpoint-80000",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--333fe38360923d6c/checkpoint-150000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                    "distil_temperature": 2.0,
                    "eval_with_current_patch_params": false,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "gelu_patch": false,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "layer_norm_patch": false,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "linear_min_parameters": 0.005,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "qat": false,
                    "qconfig": "default",
                    "regularization": "",
                    "regularization_final_lambda": 0,
                    "rewind_model_name_or_path": null
                },
                "speed": {
                    "cuda_eval_elapsed_time": 22.43720558166504,
                    "eval_elapsed_time": 26.837573747034185
                },
                "speedup": 1.966887161527097,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 769536,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2342400,
                            "linear_total": 7077888,
                            "nnz": 2349045,
                            "total": 7087104
                        },
                        "1": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 973824,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1956864,
                            "linear_total": 7077888,
                            "nnz": 1963066,
                            "total": 7086528
                        },
                        "10": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 175104,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1354752,
                            "linear_total": 7077888,
                            "nnz": 1360626,
                            "total": 7086720
                        },
                        "11": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 264192,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1443840,
                            "linear_total": 7077888,
                            "nnz": 1449772,
                            "total": 7086720
                        },
                        "2": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1055232,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2824704,
                            "linear_total": 7077888,
                            "nnz": 2831727,
                            "total": 7087296
                        },
                        "3": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1092096,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2861568,
                            "linear_total": 7077888,
                            "nnz": 2868615,
                            "total": 7087296
                        },
                        "4": {
                            "linear_attention_nnz": 2162688,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1058304,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3220992,
                            "linear_total": 7077888,
                            "nnz": 3228401,
                            "total": 7087680
                        },
                        "5": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1004544,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2970624,
                            "linear_total": 7077888,
                            "nnz": 2977806,
                            "total": 7087488
                        },
                        "6": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 768000,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2734080,
                            "linear_total": 7077888,
                            "nnz": 2741108,
                            "total": 7087488
                        },
                        "7": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 577536,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2150400,
                            "linear_total": 7077888,
                            "nnz": 2156920,
                            "total": 7087104
                        },
                        "8": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 331776,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1904640,
                            "linear_total": 7077888,
                            "nnz": 1911000,
                            "total": 7087104
                        },
                        "9": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 138240,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1514496,
                            "linear_total": 7077888,
                            "nnz": 1520538,
                            "total": 7086912
                        }
                    },
                    "linear_nnz": 27279360,
                    "linear_sparsity": 67.88194444444444,
                    "linear_total": 84934656,
                    "nnz": 51197346,
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            5,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            3,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            2
                        ],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            2,
                            3
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "total": 108884162,
                    "total_sparsity": 52.97998803535816
                },
                "training_args": {
                    "_n_gpu": 1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "length_column_name": "length",
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--333fe38360923d6c",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "mp_parameters": "",
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--333fe38360923d6c",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": [
                        "tensorboard",
                        "wandb"
                    ],
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--333fe38360923d6c",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": [],
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 75.45359715227984,
            "fill_rate": 0.3211805555555556,
            "speedup": 1.966887161527097
        },
        {
            "annotate": "32",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            5,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            3,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            2
                        ],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            2,
                            3
                        ],
                        "7": [
                            3,
                            11,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "transformers_version": "4.5.1",
                    "type_vocab_size": 2,
                    "use_cache": true,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "HasAns_exact": 72.13225371120107,
                    "HasAns_f1": 78.56627936550444,
                    "HasAns_total": 5928,
                    "NoAns_exact": 72.27922624053826,
                    "NoAns_f1": 72.27922624053826,
                    "NoAns_total": 5945,
                    "best_exact": 72.20584519498021,
                    "best_exact_thresh": 0.0,
                    "best_f1": 75.418251838517,
                    "best_f1_thresh": 0.0,
                    "exact": 72.20584519498021,
                    "f1": 75.41825183851692,
                    "total": 11873
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--333fe38360923d6c/checkpoint-82350",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--333fe38360923d6c/checkpoint-150000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                    "distil_temperature": 2.0,
                    "eval_with_current_patch_params": false,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "gelu_patch": false,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "layer_norm_patch": false,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "linear_min_parameters": 0.005,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "qat": false,
                    "qconfig": "default",
                    "regularization": "",
                    "regularization_final_lambda": 0,
                    "rewind_model_name_or_path": null
                },
                "speed": {
                    "cuda_eval_elapsed_time": 22.428810897827148,
                    "eval_elapsed_time": 26.791726927971467
                },
                "speedup": 1.967623330552778,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 769536,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2342400,
                            "linear_total": 7077888,
                            "nnz": 2349045,
                            "total": 7087104
                        },
                        "1": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 973824,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1956864,
                            "linear_total": 7077888,
                            "nnz": 1963066,
                            "total": 7086528
                        },
                        "10": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 175104,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1354752,
                            "linear_total": 7077888,
                            "nnz": 1360626,
                            "total": 7086720
                        },
                        "11": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 264192,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1443840,
                            "linear_total": 7077888,
                            "nnz": 1449772,
                            "total": 7086720
                        },
                        "2": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1055232,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2824704,
                            "linear_total": 7077888,
                            "nnz": 2831727,
                            "total": 7087296
                        },
                        "3": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1092096,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2861568,
                            "linear_total": 7077888,
                            "nnz": 2868615,
                            "total": 7087296
                        },
                        "4": {
                            "linear_attention_nnz": 2162688,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1058304,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3220992,
                            "linear_total": 7077888,
                            "nnz": 3228401,
                            "total": 7087680
                        },
                        "5": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1004544,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2970624,
                            "linear_total": 7077888,
                            "nnz": 2977806,
                            "total": 7087488
                        },
                        "6": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 768000,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2734080,
                            "linear_total": 7077888,
                            "nnz": 2741108,
                            "total": 7087488
                        },
                        "7": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 577536,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2150400,
                            "linear_total": 7077888,
                            "nnz": 2156920,
                            "total": 7087104
                        },
                        "8": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 331776,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1904640,
                            "linear_total": 7077888,
                            "nnz": 1911000,
                            "total": 7087104
                        },
                        "9": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 138240,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1514496,
                            "linear_total": 7077888,
                            "nnz": 1520538,
                            "total": 7086912
                        }
                    },
                    "linear_nnz": 27279360,
                    "linear_sparsity": 67.88194444444444,
                    "linear_total": 84934656,
                    "nnz": 51197346,
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            5,
                            6
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8
                        ],
                        "11": [
                            3,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            2
                        ],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            2,
                            3
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            8,
                            3,
                            4
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "total": 108884162,
                    "total_sparsity": 52.97998803535816
                },
                "training_args": {
                    "_n_gpu": 1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "length_column_name": "length",
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--333fe38360923d6c",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "mp_parameters": "",
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--333fe38360923d6c",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": [
                        "tensorboard",
                        "wandb"
                    ],
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--333fe38360923d6c",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": [],
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 75.41825183851692,
            "fill_rate": 0.3211805555555556,
            "speedup": 1.967623330552778
        },
        {
            "annotate": "40",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            4
                        ],
                        "1": [
                            0,
                            8,
                            2,
                            5
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6
                        ],
                        "11": [
                            3,
                            11,
                            5,
                            7
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            3
                        ],
                        "7": [
                            3,
                            11,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            8
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "transformers_version": "4.5.1",
                    "type_vocab_size": 2,
                    "use_cache": true,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "HasAns_exact": 72.60458839406208,
                    "HasAns_f1": 78.74976951058292,
                    "HasAns_total": 5928,
                    "NoAns_exact": 75.23969722455845,
                    "NoAns_f1": 75.23969722455845,
                    "NoAns_total": 5945,
                    "best_exact": 73.92402931019961,
                    "best_exact_thresh": 0.0,
                    "best_f1": 76.99222047155187,
                    "best_f1_thresh": 0.0,
                    "exact": 73.92402931019961,
                    "f1": 76.9922204715519,
                    "total": 11873
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de/checkpoint-62500",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de/checkpoint-135000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                    "distil_temperature": 2.0,
                    "eval_with_current_patch_params": false,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "gelu_patch": false,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "layer_norm_patch": false,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "linear_min_parameters": 0.005,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "qat": false,
                    "qconfig": "default",
                    "regularization": "",
                    "regularization_final_lambda": 0,
                    "rewind_model_name_or_path": null
                },
                "speed": {
                    "cuda_eval_elapsed_time": 26.13464465332031,
                    "eval_elapsed_time": 30.81871720601339
                },
                "speedup": 1.6886187734530513,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 2162688,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1434624,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3597312,
                            "linear_total": 7077888,
                            "nnz": 3604966,
                            "total": 7087680
                        },
                        "1": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1509888,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3082752,
                            "linear_total": 7077888,
                            "nnz": 3089879,
                            "total": 7087104
                        },
                        "10": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 251904,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1824768,
                            "linear_total": 7077888,
                            "nnz": 1831076,
                            "total": 7087104
                        },
                        "11": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 417792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1990656,
                            "linear_total": 7077888,
                            "nnz": 1997072,
                            "total": 7087104
                        },
                        "2": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1566720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3532800,
                            "linear_total": 7077888,
                            "nnz": 3540348,
                            "total": 7087488
                        },
                        "3": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1681920,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3451392,
                            "linear_total": 7077888,
                            "nnz": 3458823,
                            "total": 7087296
                        },
                        "4": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1637376,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3996672,
                            "linear_total": 7077888,
                            "nnz": 4004650,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1532928,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3499008,
                            "linear_total": 7077888,
                            "nnz": 3506534,
                            "total": 7087488
                        },
                        "6": {
                            "linear_attention_nnz": 2162688,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1167360,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3330048,
                            "linear_total": 7077888,
                            "nnz": 3337528,
                            "total": 7087680
                        },
                        "7": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 892416,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2465280,
                            "linear_total": 7077888,
                            "nnz": 2472005,
                            "total": 7087104
                        },
                        "8": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 473088,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2439168,
                            "linear_total": 7077888,
                            "nnz": 2446004,
                            "total": 7087488
                        },
                        "9": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 222720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1598976,
                            "linear_total": 7077888,
                            "nnz": 1605073,
                            "total": 7086912
                        }
                    },
                    "linear_nnz": 34808832,
                    "linear_sparsity": 59.01692708333333,
                    "linear_total": 84934656,
                    "nnz": 58732680,
                    "pruned_heads": {
                        "0": [
                            4
                        ],
                        "1": [
                            0,
                            8,
                            2,
                            5
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6
                        ],
                        "11": [
                            11,
                            3,
                            5,
                            7
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            3
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            8
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "total": 108887042,
                    "total_sparsity": 46.060909616775156
                },
                "training_args": {
                    "_n_gpu": 1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "length_column_name": "length",
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "mp_parameters": "",
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": [
                        "tensorboard",
                        "wandb"
                    ],
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": [],
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 76.9922204715519,
            "fill_rate": 0.40983072916666674,
            "speedup": 1.6886187734530513
        },
        {
            "annotate": "40",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            4
                        ],
                        "1": [
                            0,
                            8,
                            2,
                            5
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6
                        ],
                        "11": [
                            3,
                            11,
                            5,
                            7
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            3
                        ],
                        "7": [
                            3,
                            11,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            8
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "transformers_version": "4.5.1",
                    "type_vocab_size": 2,
                    "use_cache": true,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "HasAns_exact": 72.52024291497976,
                    "HasAns_f1": 78.86454940318372,
                    "HasAns_total": 5928,
                    "NoAns_exact": 74.41547518923466,
                    "NoAns_f1": 74.41547518923466,
                    "NoAns_total": 5945,
                    "best_exact": 73.46921586793565,
                    "best_exact_thresh": 0.0,
                    "best_f1": 76.6368271592751,
                    "best_f1_thresh": 0.0,
                    "exact": 73.46921586793565,
                    "f1": 76.6368271592751,
                    "total": 11873
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de/checkpoint-70000",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de/checkpoint-135000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                    "distil_temperature": 2.0,
                    "eval_with_current_patch_params": false,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "gelu_patch": false,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "layer_norm_patch": false,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "linear_min_parameters": 0.005,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "qat": false,
                    "qconfig": "default",
                    "regularization": "",
                    "regularization_final_lambda": 0,
                    "rewind_model_name_or_path": null
                },
                "speed": {
                    "cuda_eval_elapsed_time": 26.144072219848635,
                    "eval_elapsed_time": 30.55245121801272
                },
                "speedup": 1.688009856613554,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 2162688,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1434624,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3597312,
                            "linear_total": 7077888,
                            "nnz": 3604966,
                            "total": 7087680
                        },
                        "1": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1509888,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3082752,
                            "linear_total": 7077888,
                            "nnz": 3089879,
                            "total": 7087104
                        },
                        "10": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 251904,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1824768,
                            "linear_total": 7077888,
                            "nnz": 1831076,
                            "total": 7087104
                        },
                        "11": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 417792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1990656,
                            "linear_total": 7077888,
                            "nnz": 1997072,
                            "total": 7087104
                        },
                        "2": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1566720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3532800,
                            "linear_total": 7077888,
                            "nnz": 3540348,
                            "total": 7087488
                        },
                        "3": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1681920,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3451392,
                            "linear_total": 7077888,
                            "nnz": 3458823,
                            "total": 7087296
                        },
                        "4": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1637376,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3996672,
                            "linear_total": 7077888,
                            "nnz": 4004650,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1532928,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3499008,
                            "linear_total": 7077888,
                            "nnz": 3506534,
                            "total": 7087488
                        },
                        "6": {
                            "linear_attention_nnz": 2162688,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1167360,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3330048,
                            "linear_total": 7077888,
                            "nnz": 3337528,
                            "total": 7087680
                        },
                        "7": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 892416,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2465280,
                            "linear_total": 7077888,
                            "nnz": 2472005,
                            "total": 7087104
                        },
                        "8": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 473088,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2439168,
                            "linear_total": 7077888,
                            "nnz": 2446004,
                            "total": 7087488
                        },
                        "9": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 222720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1598976,
                            "linear_total": 7077888,
                            "nnz": 1605073,
                            "total": 7086912
                        }
                    },
                    "linear_nnz": 34808832,
                    "linear_sparsity": 59.01692708333333,
                    "linear_total": 84934656,
                    "nnz": 58732680,
                    "pruned_heads": {
                        "0": [
                            4
                        ],
                        "1": [
                            0,
                            8,
                            2,
                            5
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6
                        ],
                        "11": [
                            11,
                            3,
                            5,
                            7
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            3
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            8
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "total": 108887042,
                    "total_sparsity": 46.060909616775156
                },
                "training_args": {
                    "_n_gpu": 1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "length_column_name": "length",
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "mp_parameters": "",
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": [
                        "tensorboard",
                        "wandb"
                    ],
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": [],
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 76.6368271592751,
            "fill_rate": 0.40983072916666674,
            "speedup": 1.688009856613554
        },
        {
            "annotate": "40",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            4
                        ],
                        "1": [
                            0,
                            8,
                            2,
                            5
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6
                        ],
                        "11": [
                            3,
                            11,
                            5,
                            7
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            3
                        ],
                        "7": [
                            3,
                            11,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            8
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "transformers_version": "4.5.1",
                    "type_vocab_size": 2,
                    "use_cache": true,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "HasAns_exact": 72.35155195681511,
                    "HasAns_f1": 79.01066967291747,
                    "HasAns_total": 5928,
                    "NoAns_exact": 73.82674516400337,
                    "NoAns_f1": 73.82674516400337,
                    "NoAns_total": 5945,
                    "best_exact": 73.09020466604902,
                    "best_exact_thresh": 0.0,
                    "best_f1": 76.41499619481645,
                    "best_f1_thresh": 0.0,
                    "exact": 73.09020466604902,
                    "f1": 76.41499619481642,
                    "total": 11873
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de/checkpoint-77500",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de/checkpoint-135000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                    "distil_temperature": 2.0,
                    "eval_with_current_patch_params": false,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "gelu_patch": false,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "layer_norm_patch": false,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "linear_min_parameters": 0.005,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "qat": false,
                    "qconfig": "default",
                    "regularization": "",
                    "regularization_final_lambda": 0,
                    "rewind_model_name_or_path": null
                },
                "speed": {
                    "cuda_eval_elapsed_time": 26.154957778930665,
                    "eval_elapsed_time": 30.775646347901784
                },
                "speedup": 1.687307315581734,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 2162688,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1434624,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3597312,
                            "linear_total": 7077888,
                            "nnz": 3604966,
                            "total": 7087680
                        },
                        "1": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1509888,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3082752,
                            "linear_total": 7077888,
                            "nnz": 3089879,
                            "total": 7087104
                        },
                        "10": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 251904,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1824768,
                            "linear_total": 7077888,
                            "nnz": 1831076,
                            "total": 7087104
                        },
                        "11": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 417792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1990656,
                            "linear_total": 7077888,
                            "nnz": 1997072,
                            "total": 7087104
                        },
                        "2": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1566720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3532800,
                            "linear_total": 7077888,
                            "nnz": 3540348,
                            "total": 7087488
                        },
                        "3": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1681920,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3451392,
                            "linear_total": 7077888,
                            "nnz": 3458823,
                            "total": 7087296
                        },
                        "4": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1637376,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3996672,
                            "linear_total": 7077888,
                            "nnz": 4004650,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1532928,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3499008,
                            "linear_total": 7077888,
                            "nnz": 3506534,
                            "total": 7087488
                        },
                        "6": {
                            "linear_attention_nnz": 2162688,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1167360,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3330048,
                            "linear_total": 7077888,
                            "nnz": 3337528,
                            "total": 7087680
                        },
                        "7": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 892416,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2465280,
                            "linear_total": 7077888,
                            "nnz": 2472005,
                            "total": 7087104
                        },
                        "8": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 473088,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2439168,
                            "linear_total": 7077888,
                            "nnz": 2446004,
                            "total": 7087488
                        },
                        "9": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 222720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1598976,
                            "linear_total": 7077888,
                            "nnz": 1605073,
                            "total": 7086912
                        }
                    },
                    "linear_nnz": 34808832,
                    "linear_sparsity": 59.01692708333333,
                    "linear_total": 84934656,
                    "nnz": 58732680,
                    "pruned_heads": {
                        "0": [
                            4
                        ],
                        "1": [
                            0,
                            8,
                            2,
                            5
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6
                        ],
                        "11": [
                            11,
                            3,
                            5,
                            7
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            3
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            8
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "total": 108887042,
                    "total_sparsity": 46.060909616775156
                },
                "training_args": {
                    "_n_gpu": 1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "length_column_name": "length",
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "mp_parameters": "",
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": [
                        "tensorboard",
                        "wandb"
                    ],
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": [],
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 76.41499619481642,
            "fill_rate": 0.40983072916666674,
            "speedup": 1.687307315581734
        },
        {
            "annotate": "40",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            4
                        ],
                        "1": [
                            0,
                            8,
                            2,
                            5
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6
                        ],
                        "11": [
                            3,
                            11,
                            5,
                            7
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            3
                        ],
                        "7": [
                            3,
                            11,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            8
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "transformers_version": "4.5.1",
                    "type_vocab_size": 2,
                    "use_cache": true,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "HasAns_exact": 72.52024291497976,
                    "HasAns_f1": 79.00733071830157,
                    "HasAns_total": 5928,
                    "NoAns_exact": 73.82674516400337,
                    "NoAns_f1": 73.82674516400337,
                    "NoAns_total": 5945,
                    "best_exact": 73.17442937757939,
                    "best_exact_thresh": 0.0,
                    "best_f1": 76.41332910789967,
                    "best_f1_thresh": 0.0,
                    "exact": 73.17442937757939,
                    "f1": 76.41332910789964,
                    "total": 11873
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de/checkpoint-80000",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de/checkpoint-135000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                    "distil_temperature": 2.0,
                    "eval_with_current_patch_params": false,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "gelu_patch": false,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "layer_norm_patch": false,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "linear_min_parameters": 0.005,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "qat": false,
                    "qconfig": "default",
                    "regularization": "",
                    "regularization_final_lambda": 0,
                    "rewind_model_name_or_path": null
                },
                "speed": {
                    "cuda_eval_elapsed_time": 26.134426086425783,
                    "eval_elapsed_time": 30.504467725055292
                },
                "speedup": 1.688632895674834,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 2162688,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1434624,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3597312,
                            "linear_total": 7077888,
                            "nnz": 3604966,
                            "total": 7087680
                        },
                        "1": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1509888,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3082752,
                            "linear_total": 7077888,
                            "nnz": 3089879,
                            "total": 7087104
                        },
                        "10": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 251904,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1824768,
                            "linear_total": 7077888,
                            "nnz": 1831076,
                            "total": 7087104
                        },
                        "11": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 417792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1990656,
                            "linear_total": 7077888,
                            "nnz": 1997072,
                            "total": 7087104
                        },
                        "2": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1566720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3532800,
                            "linear_total": 7077888,
                            "nnz": 3540348,
                            "total": 7087488
                        },
                        "3": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1681920,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3451392,
                            "linear_total": 7077888,
                            "nnz": 3458823,
                            "total": 7087296
                        },
                        "4": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1637376,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3996672,
                            "linear_total": 7077888,
                            "nnz": 4004650,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1532928,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3499008,
                            "linear_total": 7077888,
                            "nnz": 3506534,
                            "total": 7087488
                        },
                        "6": {
                            "linear_attention_nnz": 2162688,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1167360,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3330048,
                            "linear_total": 7077888,
                            "nnz": 3337528,
                            "total": 7087680
                        },
                        "7": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 892416,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2465280,
                            "linear_total": 7077888,
                            "nnz": 2472005,
                            "total": 7087104
                        },
                        "8": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 473088,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2439168,
                            "linear_total": 7077888,
                            "nnz": 2446004,
                            "total": 7087488
                        },
                        "9": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 222720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1598976,
                            "linear_total": 7077888,
                            "nnz": 1605073,
                            "total": 7086912
                        }
                    },
                    "linear_nnz": 34808832,
                    "linear_sparsity": 59.01692708333333,
                    "linear_total": 84934656,
                    "nnz": 58732680,
                    "pruned_heads": {
                        "0": [
                            4
                        ],
                        "1": [
                            0,
                            8,
                            2,
                            5
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6
                        ],
                        "11": [
                            11,
                            3,
                            5,
                            7
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            3
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            8
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "total": 108887042,
                    "total_sparsity": 46.060909616775156
                },
                "training_args": {
                    "_n_gpu": 1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "length_column_name": "length",
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "mp_parameters": "",
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": [
                        "tensorboard",
                        "wandb"
                    ],
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": [],
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 76.41332910789964,
            "fill_rate": 0.40983072916666674,
            "speedup": 1.688632895674834
        },
        {
            "annotate": "40",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            4
                        ],
                        "1": [
                            0,
                            8,
                            2,
                            5
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6
                        ],
                        "11": [
                            3,
                            11,
                            5,
                            7
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            3
                        ],
                        "7": [
                            3,
                            11,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            8
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "transformers_version": "4.5.1",
                    "type_vocab_size": 2,
                    "use_cache": true,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "HasAns_exact": 72.31781376518218,
                    "HasAns_f1": 78.78776667488243,
                    "HasAns_total": 5928,
                    "NoAns_exact": 74.02859545836837,
                    "NoAns_f1": 74.02859545836837,
                    "NoAns_total": 5945,
                    "best_exact": 73.17442937757939,
                    "best_exact_thresh": 0.0,
                    "best_f1": 76.40477392813138,
                    "best_f1_thresh": 0.0,
                    "exact": 73.17442937757939,
                    "f1": 76.40477392813135,
                    "total": 11873
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de/checkpoint-82350",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de/checkpoint-135000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                    "distil_temperature": 2.0,
                    "eval_with_current_patch_params": false,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "gelu_patch": false,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "layer_norm_patch": false,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "linear_min_parameters": 0.005,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "qat": false,
                    "qconfig": "default",
                    "regularization": "",
                    "regularization_final_lambda": 0,
                    "rewind_model_name_or_path": null
                },
                "speed": {
                    "cuda_eval_elapsed_time": 26.140411270141602,
                    "eval_elapsed_time": 30.49465876701288
                },
                "speedup": 1.688246261432368,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 2162688,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1434624,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3597312,
                            "linear_total": 7077888,
                            "nnz": 3604966,
                            "total": 7087680
                        },
                        "1": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1509888,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3082752,
                            "linear_total": 7077888,
                            "nnz": 3089879,
                            "total": 7087104
                        },
                        "10": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 251904,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1824768,
                            "linear_total": 7077888,
                            "nnz": 1831076,
                            "total": 7087104
                        },
                        "11": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 417792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1990656,
                            "linear_total": 7077888,
                            "nnz": 1997072,
                            "total": 7087104
                        },
                        "2": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1566720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3532800,
                            "linear_total": 7077888,
                            "nnz": 3540348,
                            "total": 7087488
                        },
                        "3": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1681920,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3451392,
                            "linear_total": 7077888,
                            "nnz": 3458823,
                            "total": 7087296
                        },
                        "4": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1637376,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3996672,
                            "linear_total": 7077888,
                            "nnz": 4004650,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1532928,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3499008,
                            "linear_total": 7077888,
                            "nnz": 3506534,
                            "total": 7087488
                        },
                        "6": {
                            "linear_attention_nnz": 2162688,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1167360,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3330048,
                            "linear_total": 7077888,
                            "nnz": 3337528,
                            "total": 7087680
                        },
                        "7": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 892416,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2465280,
                            "linear_total": 7077888,
                            "nnz": 2472005,
                            "total": 7087104
                        },
                        "8": {
                            "linear_attention_nnz": 1966080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 473088,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2439168,
                            "linear_total": 7077888,
                            "nnz": 2446004,
                            "total": 7087488
                        },
                        "9": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 222720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1598976,
                            "linear_total": 7077888,
                            "nnz": 1605073,
                            "total": 7086912
                        }
                    },
                    "linear_nnz": 34808832,
                    "linear_sparsity": 59.01692708333333,
                    "linear_total": 84934656,
                    "nnz": 58732680,
                    "pruned_heads": {
                        "0": [
                            4
                        ],
                        "1": [
                            0,
                            8,
                            2,
                            5
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6
                        ],
                        "11": [
                            11,
                            3,
                            5,
                            7
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            3
                        ],
                        "7": [
                            11,
                            3,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            8
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "total": 108887042,
                    "total_sparsity": 46.060909616775156
                },
                "training_args": {
                    "_n_gpu": 1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "length_column_name": "length",
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "mp_parameters": "",
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": [
                        "tensorboard",
                        "wandb"
                    ],
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": [],
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 76.40477392813135,
            "fill_rate": 0.40983072916666674,
            "speedup": 1.688246261432368
        },
        {
            "annotate": "25",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            3,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            8,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            3,
                            10,
                            2
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            4
                        ],
                        "9": [
                            1,
                            2,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "transformers_version": "4.5.1",
                    "type_vocab_size": 2,
                    "use_cache": true,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "HasAns_exact": 69.46693657219973,
                    "HasAns_f1": 76.87422371820274,
                    "HasAns_total": 5928,
                    "NoAns_exact": 69.70563498738436,
                    "NoAns_f1": 69.70563498738436,
                    "NoAns_total": 5945,
                    "best_exact": 69.58645666638591,
                    "best_exact_thresh": 0.0,
                    "best_f1": 73.28479728809114,
                    "best_f1_thresh": 0.0,
                    "exact": 69.58645666638591,
                    "f1": 73.28479728809118,
                    "total": 11873
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682/checkpoint-12500",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682/checkpoint-155000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                    "distil_temperature": 2.0,
                    "eval_with_current_patch_params": false,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "gelu_patch": false,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "layer_norm_patch": false,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "linear_min_parameters": 0.005,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "qat": false,
                    "qconfig": "default",
                    "regularization": "",
                    "regularization_final_lambda": 0,
                    "rewind_model_name_or_path": null
                },
                "speed": {
                    "cuda_eval_elapsed_time": 20.071252975463867,
                    "eval_elapsed_time": 24.46170476905536
                },
                "speedup": 2.1987392442848313,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 396288,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1575936,
                            "linear_total": 7077888,
                            "nnz": 1581954,
                            "total": 7086720
                        },
                        "1": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 560640,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1543680,
                            "linear_total": 7077888,
                            "nnz": 1549613,
                            "total": 7086528
                        },
                        "10": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 118272,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1101312,
                            "linear_total": 7077888,
                            "nnz": 1106957,
                            "total": 7086528
                        },
                        "11": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 152064,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1331712,
                            "linear_total": 7077888,
                            "nnz": 1337571,
                            "total": 7086720
                        },
                        "2": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 599040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2368512,
                            "linear_total": 7077888,
                            "nnz": 2375238,
                            "total": 7087296
                        },
                        "3": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 646656,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2416128,
                            "linear_total": 7077888,
                            "nnz": 2422885,
                            "total": 7087296
                        },
                        "4": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 672768,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2442240,
                            "linear_total": 7077888,
                            "nnz": 2449014,
                            "total": 7087296
                        },
                        "5": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 609792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1986048,
                            "linear_total": 7077888,
                            "nnz": 1992397,
                            "total": 7086912
                        },
                        "6": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 486912,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2256384,
                            "linear_total": 7077888,
                            "nnz": 2263037,
                            "total": 7087296
                        },
                        "7": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 374784,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1751040,
                            "linear_total": 7077888,
                            "nnz": 1757236,
                            "total": 7086912
                        },
                        "8": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 224256,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1993728,
                            "linear_total": 7077888,
                            "nnz": 2000210,
                            "total": 7087296
                        },
                        "9": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 101376,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1281024,
                            "linear_total": 7077888,
                            "nnz": 1286850,
                            "total": 7086720
                        }
                    },
                    "linear_nnz": 22047744,
                    "linear_sparsity": 74.04152199074075,
                    "linear_total": 84934656,
                    "nnz": 45961684,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            3,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            8,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            10,
                            2,
                            3
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            4
                        ],
                        "9": [
                            1,
                            2,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "total": 108882242,
                    "total_sparsity": 57.78771344550382
                },
                "training_args": {
                    "_n_gpu": 1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "length_column_name": "length",
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "mp_parameters": "",
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": [
                        "tensorboard",
                        "wandb"
                    ],
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": [],
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 73.28479728809118,
            "fill_rate": 0.25958478009259256,
            "speedup": 2.1987392442848313
        },
        {
            "annotate": "25",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            3,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            8,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            3,
                            10,
                            2
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            4
                        ],
                        "9": [
                            1,
                            2,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "transformers_version": "4.5.1",
                    "type_vocab_size": 2,
                    "use_cache": true,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "HasAns_exact": 67.45951417004049,
                    "HasAns_f1": 74.5927984741278,
                    "HasAns_total": 5928,
                    "NoAns_exact": 72.46425567703953,
                    "NoAns_f1": 72.46425567703953,
                    "NoAns_total": 5945,
                    "best_exact": 69.96546786827255,
                    "best_exact_thresh": 0.0,
                    "best_f1": 73.52700323040777,
                    "best_f1_thresh": 0.0,
                    "exact": 69.96546786827255,
                    "f1": 73.52700323040777,
                    "total": 11873
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682/checkpoint-5000",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682/checkpoint-155000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                    "distil_temperature": 2.0,
                    "eval_with_current_patch_params": false,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "gelu_patch": false,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "layer_norm_patch": false,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "linear_min_parameters": 0.005,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "qat": false,
                    "qconfig": "default",
                    "regularization": "",
                    "regularization_final_lambda": 0,
                    "rewind_model_name_or_path": null
                },
                "speed": {
                    "cuda_eval_elapsed_time": 20.475008758544924,
                    "eval_elapsed_time": 24.819655006052926
                },
                "speedup": 2.1553813294806785,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 396288,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1575936,
                            "linear_total": 7077888,
                            "nnz": 1581954,
                            "total": 7086720
                        },
                        "1": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 560640,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1543680,
                            "linear_total": 7077888,
                            "nnz": 1549613,
                            "total": 7086528
                        },
                        "10": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 118272,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1101312,
                            "linear_total": 7077888,
                            "nnz": 1106957,
                            "total": 7086528
                        },
                        "11": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 152064,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1331712,
                            "linear_total": 7077888,
                            "nnz": 1337571,
                            "total": 7086720
                        },
                        "2": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 599040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2368512,
                            "linear_total": 7077888,
                            "nnz": 2375238,
                            "total": 7087296
                        },
                        "3": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 646656,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2416128,
                            "linear_total": 7077888,
                            "nnz": 2422885,
                            "total": 7087296
                        },
                        "4": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 672768,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2442240,
                            "linear_total": 7077888,
                            "nnz": 2449014,
                            "total": 7087296
                        },
                        "5": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 609792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1986048,
                            "linear_total": 7077888,
                            "nnz": 1992397,
                            "total": 7086912
                        },
                        "6": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 486912,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2256384,
                            "linear_total": 7077888,
                            "nnz": 2263037,
                            "total": 7087296
                        },
                        "7": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 374784,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1751040,
                            "linear_total": 7077888,
                            "nnz": 1757236,
                            "total": 7086912
                        },
                        "8": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 224256,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1993728,
                            "linear_total": 7077888,
                            "nnz": 2000210,
                            "total": 7087296
                        },
                        "9": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 101376,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1281024,
                            "linear_total": 7077888,
                            "nnz": 1286850,
                            "total": 7086720
                        }
                    },
                    "linear_nnz": 22047744,
                    "linear_sparsity": 74.04152199074075,
                    "linear_total": 84934656,
                    "nnz": 45961684,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            3,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            8,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            10,
                            2,
                            3
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            4
                        ],
                        "9": [
                            1,
                            2,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "total": 108882242,
                    "total_sparsity": 57.78771344550382
                },
                "training_args": {
                    "_n_gpu": 1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "length_column_name": "length",
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "mp_parameters": "",
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": [
                        "tensorboard",
                        "wandb"
                    ],
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": [],
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 73.52700323040777,
            "fill_rate": 0.25958478009259256,
            "speedup": 2.1553813294806785
        },
        {
            "annotate": "25",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            3,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            8,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            3,
                            10,
                            2
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            4
                        ],
                        "9": [
                            1,
                            2,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "transformers_version": "4.5.1",
                    "type_vocab_size": 2,
                    "use_cache": true,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "HasAns_exact": 67.22334682860999,
                    "HasAns_f1": 74.54878992793641,
                    "HasAns_total": 5928,
                    "NoAns_exact": 72.26240538267452,
                    "NoAns_f1": 72.26240538267452,
                    "NoAns_total": 5945,
                    "best_exact": 69.74648361829361,
                    "best_exact_thresh": 0.0,
                    "best_f1": 73.40396080963606,
                    "best_f1_thresh": 0.0,
                    "exact": 69.74648361829361,
                    "f1": 73.40396080963602,
                    "total": 11873
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682/checkpoint-7500",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682/checkpoint-155000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                    "distil_temperature": 2.0,
                    "eval_with_current_patch_params": false,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "gelu_patch": false,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "layer_norm_patch": false,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "linear_min_parameters": 0.005,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "qat": false,
                    "qconfig": "default",
                    "regularization": "",
                    "regularization_final_lambda": 0,
                    "rewind_model_name_or_path": null
                },
                "speed": {
                    "cuda_eval_elapsed_time": 20.078421813964844,
                    "eval_elapsed_time": 24.476557928021066
                },
                "speedup": 2.197954202178729,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 396288,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1575936,
                            "linear_total": 7077888,
                            "nnz": 1581954,
                            "total": 7086720
                        },
                        "1": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 560640,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1543680,
                            "linear_total": 7077888,
                            "nnz": 1549613,
                            "total": 7086528
                        },
                        "10": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 118272,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1101312,
                            "linear_total": 7077888,
                            "nnz": 1106957,
                            "total": 7086528
                        },
                        "11": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 152064,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1331712,
                            "linear_total": 7077888,
                            "nnz": 1337571,
                            "total": 7086720
                        },
                        "2": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 599040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2368512,
                            "linear_total": 7077888,
                            "nnz": 2375238,
                            "total": 7087296
                        },
                        "3": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 646656,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2416128,
                            "linear_total": 7077888,
                            "nnz": 2422885,
                            "total": 7087296
                        },
                        "4": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 672768,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2442240,
                            "linear_total": 7077888,
                            "nnz": 2449014,
                            "total": 7087296
                        },
                        "5": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 609792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1986048,
                            "linear_total": 7077888,
                            "nnz": 1992397,
                            "total": 7086912
                        },
                        "6": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 486912,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2256384,
                            "linear_total": 7077888,
                            "nnz": 2263037,
                            "total": 7087296
                        },
                        "7": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 374784,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1751040,
                            "linear_total": 7077888,
                            "nnz": 1757236,
                            "total": 7086912
                        },
                        "8": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 224256,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1993728,
                            "linear_total": 7077888,
                            "nnz": 2000210,
                            "total": 7087296
                        },
                        "9": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 101376,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1281024,
                            "linear_total": 7077888,
                            "nnz": 1286850,
                            "total": 7086720
                        }
                    },
                    "linear_nnz": 22047744,
                    "linear_sparsity": 74.04152199074075,
                    "linear_total": 84934656,
                    "nnz": 45961684,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            5,
                            6,
                            7
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            3,
                            5,
                            7,
                            8,
                            10,
                            11
                        ],
                        "2": [
                            8,
                            4,
                            7
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [
                            8,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            6,
                            7,
                            11
                        ],
                        "6": [
                            10,
                            2,
                            3
                        ],
                        "7": [
                            1,
                            3,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            8,
                            4
                        ],
                        "9": [
                            1,
                            2,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "total": 108882242,
                    "total_sparsity": 57.78771344550382
                },
                "training_args": {
                    "_n_gpu": 1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "length_column_name": "length",
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "mp_parameters": "",
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": [
                        "tensorboard",
                        "wandb"
                    ],
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": [],
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 73.40396080963602,
            "fill_rate": 0.25958478009259256,
            "speedup": 2.197954202178729
        }
    ]
}